fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:225]: Timed out after 600.000s.
public ingress should be reachable from outside the VNet despite private KAS
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:223 with:
public ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get "https://console-openshift-console.apps.aro.private-kas.0ju5.j8150016.hcp.osadev.cloud": context deadline exceeded
Unexpected error:
    <*url.Error | 0x36a4b7cb55f0>: 
    Get "https://console-openshift-console.apps.aro.private-kas.0ju5.j8150016.hcp.osadev.cloud": context deadline exceeded
    {
        Op: "Get",
        URL: "https://console-openshift-console.apps.aro.private-kas.0ju5.j8150016.hcp.osadev.cloud",
        Err: <context.deadlineExceededError>{},
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:225]: Timed out after 600.000s.
public ingress should be reachable from outside the VNet despite private KAS
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:223 with:
public ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get "https://console-openshift-console.apps.aro.private-kas.0ju5.j8150016.hcp.osadev.cloud": context deadline exceeded
Unexpected error:
    <*url.Error | 0x36a4b7cb55f0>: 
    Get "https://console-openshift-console.apps.aro.private-kas.0ju5.j8150016.hcp.osadev.cloud": context deadline exceeded
    {
        Op: "Get",
        URL: "https://console-openshift-console.apps.aro.private-kas.0ju5.j8150016.hcp.osadev.cloud",
        Err: <context.deadlineExceededError>{},
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.001s.
Expected success, but got an error:
    <*fmt.wrapError | 0xd373b6d7240>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);
    let _endTime = datetime(2026-10-16T00:41:03.5278907Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-wscgxrnrv56t'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-16T00:41:17.7915220Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310", ActivityId:"94321735-0ea3-4275-97c8-e484b256af4b", SubActivityId:"94321735-0ea3-4275-97c8-e484b256af4b", ActivityType:"GW.Http.CallContext", ParentActivityId:"94321735-0ea3-4275-97c8-e484b256af4b", ActivityStack:"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b > GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-16T00:41:17.7915220Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\", ActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", SubActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b > GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0xd373b6616c8>{
            errs: [
                <*fmt.wrapError | 0xd373b6d7220>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-16T00:41:17.7915220Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\", ActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", SubActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b > GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0xd373b6d7200>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-16T00:41:17.7915220Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\", ActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", SubActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b > GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0xd373b6d7140>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-16T00:41:17.7915220Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\", ActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", SubActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b > GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0xd373b6d7100>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-16T00:41:17.7915220Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\", ActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", SubActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b > GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0xd3739a6a100>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.001s.
Expected success, but got an error:
    <*fmt.wrapError | 0xd373b6d7240>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);
    let _endTime = datetime(2026-10-16T00:41:03.5278907Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-wscgxrnrv56t'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-16T00:41:17.7915220Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310", ActivityId:"94321735-0ea3-4275-97c8-e484b256af4b", SubActivityId:"94321735-0ea3-4275-97c8-e484b256af4b", ActivityType:"GW.Http.CallContext", ParentActivityId:"94321735-0ea3-4275-97c8-e484b256af4b", ActivityStack:"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b > GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-16T00:41:17.7915220Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\", ActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", SubActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b > GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0xd373b6616c8>{
            errs: [
                <*fmt.wrapError | 0xd373b6d7220>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-16T00:41:17.7915220Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\", ActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", SubActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b > GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0xd373b6d7200>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-16T00:41:17.7915220Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\", ActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", SubActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b > GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0xd373b6d7140>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-16T00:41:17.7915220Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\", ActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", SubActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b > GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0xd373b6d7100>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-16T00:41:17.7915220Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\", ActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", SubActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b > GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0xd3739a6a100>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }

time=2026-10-15T23:49:29.451Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 description="Step storageclass\n Kind: Helm\n"
time=2026-10-15T23:49:29.912Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 values="map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:d3bdf5672c862ebc0a35ea497266f8c7598862b9377e076ed14ffc329be19981]"
time=2026-10-15T23:49:29.912Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1
time=2026-10-15T23:49:30.191Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 namespace=default-sc
time=2026-10-15T23:49:30.484Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 dryRun=false
time=2026-10-15T23:49:30.485Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1
time=2026-10-15T23:49:30.579Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1
time=2026-10-15T23:50:28.126Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err="failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1"
time=2026-10-15T23:50:28.126Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err="stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1"

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5bn6dj9spww9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x45cfe48c260>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5bn6dj9spww9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.01s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5bn6dj9spww9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T23:14:40.1229302Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5bn6dj9spww9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.01s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5bn6dj9spww9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T23:14:40.1229302Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x45cfe48c240>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5bn6dj9spww9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T23:14:40.1229302Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x45cfe7904b0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x45cfe48c220>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5bn6dj9spww9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T23:14:40.1229302Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x45d0010c480>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5bn6dj9spww9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x45cfe48c260>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5bn6dj9spww9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.01s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5bn6dj9spww9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T23:14:40.1229302Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5bn6dj9spww9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.01s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5bn6dj9spww9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T23:14:40.1229302Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x45cfe48c240>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5bn6dj9spww9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T23:14:40.1229302Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x45cfe7904b0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x45cfe48c220>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5bn6dj9spww9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T23:14:40.1229302Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x45d0010c480>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

time=2026-10-15T22:37:03.691Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description="Step cluster\n  Kind: ARM\n  Template: templates/svc-cluster.bicep\n  Parameters: configurations/svc-cluster.tmpl.bicepparam"
time=2026-10-15T22:37:05.354Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster
time=2026-10-15T22:37:06.762Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=bee630fb16cd45492e943a7ab4f37f451fa3266ef665a6d74f650467bd60800f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9396992-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fbee630fb16cd45492e943a7ab4f37f451fa3266ef665a6d74f650467bd60800f
time=2026-10-15T22:40:07.453Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9396992-svc/providers/Microsoft.Resources/deployments/bee630fb16cd45492e943a7ab4f37f451fa3266ef665a6d74f650467bd60800f/operationStatuses/08584095030595181402\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9396992-svc/providers/Microsoft.Resources/deployments/cluster-kpgg4tugptwaw\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9396992-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j9396992-svc\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\",\\r\\n                \\\"message\\\": \\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\"https://ah-ci01-se-j9396992-1.vault.azure.net\\\\\\\"\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool "np-1" for autoscaling cluster
Unexpected error:
    <*fmt.wrapError | 0x172a79b02020>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "autoscaling-hcp-cluster" in resourcegroup="autoscaling-cluster-tjmbtfqzrk9p" to finish creating: GET https://rp.j5936768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/166953ee-9ecb-476c-9dcf-5d9a18601d55
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/166953ee-9ecb-476c-9dcf-5d9a18601d55",
      "name": "166953ee-9ecb-476c-9dcf-5d9a18601d55",
      "status": "Failed",
      "startTime": "2026-10-15T23:11:14.489775395Z",
      "endTime": "2026-10-15T23:30:19.374154808Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine m5d0c0m5m4r5p9d-np-1-2f6qw-lqwqf: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-tjmbtfqzrk9p--managed/m5d0c0m5m4r5p9d-np-1-2f6qw-lqwqf-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-tjmbtfqzrk9p--managed/providers/Microsoft.Network/networkInterfaces/m5d0c0m5m4r5p9d-np-1-2f6qw-lqwqf-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '3' seconds after additional tokens are available.... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5936768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/166953ee-9ecb-476c-9dcf-5d9a18601d55
            correlation ID: 0f13c63d-8045-459d-a9cd-c324c682e717
            client request ID: a95fa974-534b-4390-90e8-ad21cd86c818
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool "np-1" for autoscaling cluster
Unexpected error:
    <*fmt.wrapError | 0x172a79b02020>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "autoscaling-hcp-cluster" in resourcegroup="autoscaling-cluster-tjmbtfqzrk9p" to finish creating: GET https://rp.j5936768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/166953ee-9ecb-476c-9dcf-5d9a18601d55
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/166953ee-9ecb-476c-9dcf-5d9a18601d55",
      "name": "166953ee-9ecb-476c-9dcf-5d9a18601d55",
      "status": "Failed",
      "startTime": "2026-10-15T23:11:14.489775395Z",
      "endTime": "2026-10-15T23:30:19.374154808Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine m5d0c0m5m4r5p9d-np-1-2f6qw-lqwqf: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-tjmbtfqzrk9p--managed/m5d0c0m5m4r5p9d-np-1-2f6qw-lqwqf-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-tjmbtfqzrk9p--managed/providers/Microsoft.Network/networkInterfaces/m5d0c0m5m4r5p9d-np-1-2f6qw-lqwqf-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '3' seconds after additional tokens are available.... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5936768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/166953ee-9ecb-476c-9dcf-5d9a18601d55
            correlation ID: 0f13c63d-8045-459d-a9cd-c324c682e717
            client request ID: a95fa974-534b-4390-90e8-ad21cd86c818
occurred

alert fired 5 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T23:11:29Z
  Ended:       2026-10-15T23:27:26Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5967616-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfgjfkin4c231jar7eo006vk49ls0mn-r3u6k9z7l5b6z3z", pod="multus-admission-controller-6c6d8f9ddd-tctrm", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfgjfkin4c231jar7eo006vk49ls0mn-r3u6k9z7l5b6z3z/multus-admission-controller-6c6d8f9ddd-tctrm has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-15T23:11:29Z
  Ended:       2026-10-15T23:27:26Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5967616-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfgjfkin4c231jar7eo006vk49ls0mn-r3u6k9z7l5b6z3z", pod="multus-admission-controller-6c6d8f9ddd-jr5z4", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfgjfkin4c231jar7eo006vk49ls0mn-r3u6k9z7l5b6z3z/multus-admission-controller-6c6d8f9ddd-jr5z4 has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-15T23:11:29Z
  Ended:       2026-10-15T23:31:25Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5967616-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfgjfkin4c231jar7eo006vk49ls0mn-r3u6k9z7l5b6z3z", pod="multus-admission-controller-6c6d8f9ddd-t5wbj", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfgjfkin4c231jar7eo006vk49ls0mn-r3u6k9z7l5b6z3z/multus-admission-controller-6c6d8f9ddd-t5wbj has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Resolved
  Started:     2026-10-15T23:11:29Z
  Ended:       2026-10-15T23:31:25Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5967616-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfgjfkin4c231jar7eo006vk49ls0mn-r3u6k9z7l5b6z3z", pod="multus-admission-controller-6c6d8f9ddd-wg6df", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfgjfkin4c231jar7eo006vk49ls0mn-r3u6k9z7l5b6z3z/multus-admission-controller-6c6d8f9ddd-wg6df has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Resolved
  Started:     2026-10-15T23:11:29Z
  Ended:       2026-10-15T23:27:26Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5967616-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfgjfkin4c231jar7eo006vk49ls0mn-r3u6k9z7l5b6z3z", pod="multus-admission-controller-6c6d8f9ddd-cbs7b", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfgjfkin4c231jar7eo006vk49ls0mn-r3u6k9z7l5b6z3z/multus-admission-controller-6c6d8f9ddd-cbs7b has been in a non-ready state for longer than 5 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hdc4wgp48jcq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1598f140800>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hdc4wgp48jcq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hdc4wgp48jcq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T21:40:02.1232498Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hdc4wgp48jcq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hdc4wgp48jcq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T21:40:02.1232498Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1598f1407e0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hdc4wgp48jcq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T21:40:02.1232498Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1598e0be690>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1598f1407c0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hdc4wgp48jcq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T21:40:02.1232498Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x15990611c20>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hdc4wgp48jcq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1598f140800>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hdc4wgp48jcq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hdc4wgp48jcq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T21:40:02.1232498Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hdc4wgp48jcq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hdc4wgp48jcq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T21:40:02.1232498Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1598f1407e0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hdc4wgp48jcq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T21:40:02.1232498Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1598e0be690>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1598f1407c0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hdc4wgp48jcq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T21:40:02.1232498Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x15990611c20>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool "nodepool-128GiB" with 128GiB osDisk
Unexpected error:
    <*fmt.wrapError | 0x2d528da08240>: 
    failed to create NodePool nodepool-128GiB: failed waiting for nodepool="nodepool-128GiB" for cluster "hcp-cluster-np-128" in resourcegroup="clusternp128-555prpnqd6q9" to finish creating: GET https://rp.j4481792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68716f1-756f-4cdd-a508-f1a64147dcc7
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68716f1-756f-4cdd-a508-f1a64147dcc7",
      "name": "e68716f1-756f-4cdd-a508-f1a64147dcc7",
      "status": "Failed",
      "startTime": "2026-10-15T22:00:18.084367751Z",
      "endTime": "2026-10-15T22:29:20.241927727Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-9l6x8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-555prpnqd6q9--managed/providers/Microsoft.Compute/virtualMachines/u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-9l6x8 to exist\nMachine u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-tcn2v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-555prpnqd6q9--managed/providers/Microsoft.Compute/virtualMachines/u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-tcn2v to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-9l6x8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-555prpnqd6q9--managed/providers/Microsoft.Compute/virtualMachines/u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-9l6x8 to exist\nMachine u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-tcn2v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-555prpnqd6q9--managed/providers/Microsoft.Compute/virtualMachines/u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-tcn2v to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4481792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68716f1-756f-4cdd-a508-f1a64147dcc7
            correlation ID: 7e1b7dd4-0022-48fb-91be-ab56afbfcaf9
            client request ID: eaae09a9-3401-4404-aa69-a9f272cf4e2d
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool "nodepool-128GiB" with 128GiB osDisk
Unexpected error:
    <*fmt.wrapError | 0x2d528da08240>: 
    failed to create NodePool nodepool-128GiB: failed waiting for nodepool="nodepool-128GiB" for cluster "hcp-cluster-np-128" in resourcegroup="clusternp128-555prpnqd6q9" to finish creating: GET https://rp.j4481792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68716f1-756f-4cdd-a508-f1a64147dcc7
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68716f1-756f-4cdd-a508-f1a64147dcc7",
      "name": "e68716f1-756f-4cdd-a508-f1a64147dcc7",
      "status": "Failed",
      "startTime": "2026-10-15T22:00:18.084367751Z",
      "endTime": "2026-10-15T22:29:20.241927727Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-9l6x8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-555prpnqd6q9--managed/providers/Microsoft.Compute/virtualMachines/u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-9l6x8 to exist\nMachine u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-tcn2v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-555prpnqd6q9--managed/providers/Microsoft.Compute/virtualMachines/u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-tcn2v to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-9l6x8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-555prpnqd6q9--managed/providers/Microsoft.Compute/virtualMachines/u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-9l6x8 to exist\nMachine u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-tcn2v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-555prpnqd6q9--managed/providers/Microsoft.Compute/virtualMachines/u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-tcn2v to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4481792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68716f1-756f-4cdd-a508-f1a64147dcc7
            correlation ID: 7e1b7dd4-0022-48fb-91be-ab56afbfcaf9
            client request ID: eaae09a9-3401-4404-aa69-a9f272cf4e2d
occurred

time=2026-10-15T20:48:50.569Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description="Step cluster\n  Kind: ARM\n  Template: templates/svc-cluster.bicep\n  Parameters: configurations/svc-cluster.tmpl.bicepparam"
time=2026-10-15T20:48:52.496Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster
time=2026-10-15T20:48:54.249Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=8276bf3e2ac415cdaf52404f9c09132aafef64d9fdc663f4e7aca3248fabbce7 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3682688-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F8276bf3e2ac415cdaf52404f9c09132aafef64d9fdc663f4e7aca3248fabbce7
time=2026-10-15T20:51:24.632Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3682688-svc/providers/Microsoft.Resources/deployments/8276bf3e2ac415cdaf52404f9c09132aafef64d9fdc663f4e7aca3248fabbce7/operationStatuses/08584095095521617911\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3682688-svc/providers/Microsoft.Resources/deployments/cluster-a22genk7v3fe6\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3682688-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j3682688-svc\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\",\\r\\n                \\\"message\\\": \\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\"https://ah-ci01-se-j3682688-1.vault.azure.net\\\\\\\"\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling
Unexpected error:
    <*fmt.wrapError | 0x12c08c2c7ca0>: 
    failed to create NodePool autoscale-noaz: failed waiting for nodepool="autoscale-noaz" for cluster "np-autoscale-cluster" in resourcegroup="np-autoscaling-ftltmptlj2wv" to finish creating: GET https://rp.j5160960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a570a939-3665-4ef5-997f-a303cdd226fe
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a570a939-3665-4ef5-997f-a303cdd226fe",
      "name": "a570a939-3665-4ef5-997f-a303cdd226fe",
      "status": "Failed",
      "startTime": "2026-10-15T21:24:37.707389453Z",
      "endTime": "2026-10-15T21:53:37.621733074Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5160960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a570a939-3665-4ef5-997f-a303cdd226fe
            correlation ID: f3ada34b-c475-444b-90fd-a47f28fcd020
            client request ID: 8db3b288-30a6-4fd1-8aaf-660ebcafd494
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling
Unexpected error:
    <*fmt.wrapError | 0x12c08c2c7ca0>: 
    failed to create NodePool autoscale-noaz: failed waiting for nodepool="autoscale-noaz" for cluster "np-autoscale-cluster" in resourcegroup="np-autoscaling-ftltmptlj2wv" to finish creating: GET https://rp.j5160960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a570a939-3665-4ef5-997f-a303cdd226fe
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a570a939-3665-4ef5-997f-a303cdd226fe",
      "name": "a570a939-3665-4ef5-997f-a303cdd226fe",
      "status": "Failed",
      "startTime": "2026-10-15T21:24:37.707389453Z",
      "endTime": "2026-10-15T21:53:37.621733074Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5160960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a570a939-3665-4ef5-997f-a303cdd226fe
            correlation ID: f3ada34b-c475-444b-90fd-a47f28fcd020
            client request ID: 8db3b288-30a6-4fd1-8aaf-660ebcafd494
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls
Unexpected error:
    <*fmt.wrapError | 0x371fb75e40>: 
    failed to create NodePool swift-workers: failed waiting for nodepool="swift-workers" for cluster "engineering-hcp-cluster" in resourcegroup="kusto-logs-htlwf6tksvkk" to finish creating: GET https://rp.j2353536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b439425f-1c8a-4eb5-8449-8ffb64a115d8
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b439425f-1c8a-4eb5-8449-8ffb64a115d8",
      "name": "b439425f-1c8a-4eb5-8449-8ffb64a115d8",
      "status": "Failed",
      "startTime": "2026-10-15T21:32:41.543837723Z",
      "endTime": "2026-10-15T22:01:41.868424358Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine f0h5o6h1r1j7z7l-swift-workers-wnvr8-4x9jq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-htlwf6tksvkk--managed/providers/Microsoft.Compute/virtualMachines/f0h5o6h1r1j7z7l-swift-workers-wnvr8-4x9jq to exist\nMachine f0h5o6h1r1j7z7l-swift-workers-wnvr8-t8p9t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-htlwf6tksvkk--managed/providers/Microsoft.Compute/virtualMachines/f0h5o6h1r1j7z7l-swift-workers-wnvr8-t8p9t to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine f0h5o6h1r1j7z7l-swift-workers-wnvr8-4x9jq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-htlwf6tksvkk--managed/providers/Microsoft.Compute/virtualMachines/f0h5o6h1r1j7z7l-swift-workers-wnvr8-4x9jq to exist\nMachine f0h5o6h1r1j7z7l-swift-workers-wnvr8-t8p9t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-htlwf6tksvkk--managed/providers/Microsoft.Compute/virtualMachines/f0h5o6h1r1j7z7l-swift-workers-wnvr8-t8p9t to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2353536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b439425f-1c8a-4eb5-8449-8ffb64a115d8
            correlation ID: 11b13ff0-4dd4-474f-81b1-b4341cbc7ec0
            client request ID: 9a03c5e0-eb37-442c-b9ce-57bebab70a3c
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls
Unexpected error:
    <*fmt.wrapError | 0x371fb75e40>: 
    failed to create NodePool swift-workers: failed waiting for nodepool="swift-workers" for cluster "engineering-hcp-cluster" in resourcegroup="kusto-logs-htlwf6tksvkk" to finish creating: GET https://rp.j2353536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b439425f-1c8a-4eb5-8449-8ffb64a115d8
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b439425f-1c8a-4eb5-8449-8ffb64a115d8",
      "name": "b439425f-1c8a-4eb5-8449-8ffb64a115d8",
      "status": "Failed",
      "startTime": "2026-10-15T21:32:41.543837723Z",
      "endTime": "2026-10-15T22:01:41.868424358Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine f0h5o6h1r1j7z7l-swift-workers-wnvr8-4x9jq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-htlwf6tksvkk--managed/providers/Microsoft.Compute/virtualMachines/f0h5o6h1r1j7z7l-swift-workers-wnvr8-4x9jq to exist\nMachine f0h5o6h1r1j7z7l-swift-workers-wnvr8-t8p9t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-htlwf6tksvkk--managed/providers/Microsoft.Compute/virtualMachines/f0h5o6h1r1j7z7l-swift-workers-wnvr8-t8p9t to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine f0h5o6h1r1j7z7l-swift-workers-wnvr8-4x9jq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-htlwf6tksvkk--managed/providers/Microsoft.Compute/virtualMachines/f0h5o6h1r1j7z7l-swift-workers-wnvr8-4x9jq to exist\nMachine f0h5o6h1r1j7z7l-swift-workers-wnvr8-t8p9t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-htlwf6tksvkk--managed/providers/Microsoft.Compute/virtualMachines/f0h5o6h1r1j7z7l-swift-workers-wnvr8-t8p9t to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2353536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b439425f-1c8a-4eb5-8449-8ffb64a115d8
            correlation ID: 11b13ff0-4dd4-474f-81b1-b4341cbc7ec0
            client request ID: 9a03c5e0-eb37-442c-b9ce-57bebab70a3c
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:123]: Timed out after 651.749s.
Expected success, but got an error:
    <*errors.errorString | 0x48c3b8dec90>: 
    must-gather CLI test failures:
      query-basic: command failed: exit status 1
    output: �[37m[21:33:14.493]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-basic-3888395104"
    }�[0m
    �[37m[21:33:23.438]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:33:21.8572874Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\", ActivityId:\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\", SubActivityId:\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\"}, IsPermanent:false})",
      "name": "detailedInfraOrchestrationLogs"
    }�[0m
    �[37m[21:33:23.438]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "query \"backendControllerConditions\" time window 50/289 failed (KQL: let _startTime = datetime(2026-10-15T01:38:14.4931102Z);\nlet _endTime = datetime(2026-10-15T01:43:14.4931101Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-15T01:38:14.4931102Z);\nlet _endTime = datetime(2026-10-15T01:43:14.4931101Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context canceled",
      "name": "backendControllerConditions"
    }�[0m
    �[37m[21:33:23.712]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{
      "input": "/tmp/must-gather-cli-e2e-query-basic-3888395104/custom/custom-query_cosmosResourceSnapshots.jsonl",
      "output": "/tmp/must-gather-cli-e2e-query-basic-3888395104/cosmosContent"
    }�[0m
    �[37m[21:33:23.712]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-basic-3888395104/custom/custom-query_cosmosResourceSnapshots.jsonl"
    }�[0m
    �[37m[21:33:24.208]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{
      "count": 1498,
      "files": 1
    }�[0m
    �[37m[21:33:24.208]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-basic-3888395104/cosmosContent"
    }�[0m
    �[37m[21:33:24.217]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m
    �[37m[21:33:40.606]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{
      "commits": 793
    }�[0m
    �[37m[21:33:40.607]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{
      "clusterIDs": null,
      "output": "/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect"
    }�[0m
    �[37m[21:33:40.961]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{
      "clusterIDs": [
        "2tfevok49t0kjg6c48lr24m7jgn4lk62"
      ]
    }�[0m
    �[37m[21:33:41.088]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{
      "clusterNames": [
        "ci01-j8700672-mgmt-1",
        "ci01-j8700672-svc"
      ]
    }�[0m
    �[37m[21:33:41.127]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespaces": [
        "2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g"
      ]
    }�[0m
    �[37m[21:33:41.127]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{
      "managementCluster": "ci01-j8700672-mgmt-1",
      "namespaceCount": 4
    }�[0m
    �[37m[21:33:41.127]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:33:41.317]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:33:41.478]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:33:41.623]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g",
      "output": "/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g"
    }�[0m
    �[37m[21:33:43.044]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:33:21.8572874Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\", ActivityId:\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\", SubActivityId:\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\"}, IsPermanent:false})",
      "output": "/tmp/must-gather-cli-e2e-query-basic-3888395104"
    }�[0m
    �[37m[21:33:43.044]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:33:21.8572874Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\", ActivityId:\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\", SubActivityId:\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\"}, IsPermanent:false})"
    }�[0m
    
      query-skip-hcp-logs: command failed: exit status 1
    output: �[37m[21:33:43.407]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843"
    }�[0m
    �[37m[21:33:49.648]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:33:47.6716888Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\", ActivityId:\"4f13c479-3f1c-438a-b976-532417bcd103\", SubActivityId:\"4f13c479-3f1c-438a-b976-532417bcd103\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"4f13c479-3f1c-438a-b976-532417bcd103\", ActivityStack:\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\"}, IsPermanent:false})",
      "name": "detailedInfraOrchestrationLogs"
    }�[0m
    �[37m[21:33:49.648]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "query \"backendControllerConditions\" time window 47/289 failed (KQL: let _startTime = datetime(2026-10-15T01:23:43.4070857Z);\nlet _endTime = datetime(2026-10-15T01:28:43.4070856Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled",
      "name": "backendControllerConditions"
    }�[0m
    �[37m[21:33:49.744]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{
      "input": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/custom/custom-query_cosmosResourceSnapshots.jsonl",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/cosmosContent"
    }�[0m
    �[37m[21:33:49.744]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/custom/custom-query_cosmosResourceSnapshots.jsonl"
    }�[0m
    �[37m[21:33:50.271]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{
      "count": 1498,
      "files": 1
    }�[0m
    �[37m[21:33:50.271]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/cosmosContent"
    }�[0m
    �[37m[21:33:50.281]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m
    �[37m[21:34:08.647]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{
      "commits": 793
    }�[0m
    �[37m[21:34:08.648]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{
      "clusterIDs": null,
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect"
    }�[0m
    �[37m[21:34:09.212]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{
      "clusterIDs": [
        "2tfevok49t0kjg6c48lr24m7jgn4lk62"
      ]
    }�[0m
    �[37m[21:34:09.309]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{
      "clusterNames": [
        "ci01-j8700672-mgmt-1",
        "ci01-j8700672-svc"
      ]
    }�[0m
    �[37m[21:34:09.470]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespaces": [
        "2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g"
      ]
    }�[0m
    �[37m[21:34:09.470]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{
      "managementCluster": "ci01-j8700672-mgmt-1",
      "namespaceCount": 4
    }�[0m
    �[37m[21:34:09.470]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:34:09.615]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:34:10.183]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:34:10.617]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g"
    }�[0m
    �[37m[21:34:12.156]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:33:47.6716888Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\", ActivityId:\"4f13c479-3f1c-438a-b976-532417bcd103\", SubActivityId:\"4f13c479-3f1c-438a-b976-532417bcd103\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"4f13c479-3f1c-438a-b976-532417bcd103\", ActivityStack:\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\"}, IsPermanent:false})",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843"
    }�[0m
    �[37m[21:34:12.157]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:33:47.6716888Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\", ActivityId:\"4f13c479-3f1c-438a-b976-532417bcd103\", SubActivityId:\"4f13c479-3f1c-438a-b976-532417bcd103\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"4f13c479-3f1c-438a-b976-532417bcd103\", ActivityStack:\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\"}, IsPermanent:false})"
    }�[0m
    
      query-skip-kubernetes-events: command failed: exit status 1
    output: �[37m[21:34:12.559]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767"
    }�[0m
    �[37m[21:34:18.220]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:34:16.3210435Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\", ActivityId:\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\", SubActivityId:\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\"}, IsPermanent:false})",
      "name": "detailedInfraOrchestrationLogs"
    }�[0m
    �[37m[21:34:18.221]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "query \"backendControllerConditions\" time window 47/289 failed (KQL: let _startTime = datetime(2026-10-15T01:24:12.5582990Z);\nlet _endTime = datetime(2026-10-15T01:29:12.5582989Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled",
      "name": "backendControllerConditions"
    }�[0m
    �[37m[21:34:18.221]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{
      "input": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/custom/custom-query_cosmosResourceSnapshots.jsonl",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/cosmosContent"
    }�[0m
    �[37m[21:34:18.221]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/custom/custom-query_cosmosResourceSnapshots.jsonl"
    }�[0m
    �[37m[21:34:18.723]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{
      "count": 1498,
      "files": 1
    }�[0m
    �[37m[21:34:18.724]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/cosmosContent"
    }�[0m
    �[37m[21:34:18.734]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m
    �[37m[21:34:35.437]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{
      "clusterIDs": null,
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect"
    }�[0m
    �[37m[21:34:35.845]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{
      "clusterIDs": [
        "2tfevok49t0kjg6c48lr24m7jgn4lk62"
      ]
    }�[0m
    �[37m[21:34:36.194]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{
      "clusterNames": [
        "ci01-j8700672-mgmt-1",
        "ci01-j8700672-svc"
      ]
    }�[0m
    �[37m[21:34:36.313]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespaces": [
        "2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g"
      ]
    }�[0m
    �[37m[21:34:36.313]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{
      "managementCluster": "ci01-j8700672-mgmt-1",
      "namespaceCount": 4
    }�[0m
    �[37m[21:34:36.313]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:34:36.544]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:34:36.863]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:34:37.093]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g"
    }�[0m
    �[37m[21:34:38.438]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:34:16.3210435Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\", ActivityId:\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\", SubActivityId:\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\"}, IsPermanent:false})\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\n",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767"
    }�[0m
    �[37m[21:34:38.438]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:34:16.3210435Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\", ActivityId:\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\", SubActivityId:\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\"}, IsPermanent:false})\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\n"
    }�[0m
    
      query-collect-systemd-logs: command failed: exit status 1
    output: �[37m[21:34:38.772]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904"
    }�[0m
    �[37m[21:34:46.539]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:34:46.3046137Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\", ActivityId:\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\", SubActivityId:\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\", ActivityStack:\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\"}, IsPermanent:false})",
      "name": "detailedInfraOrchestrationLogs"
    }�[0m
    �[37m[21:34:46.539]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "query \"backendControllerConditions\" time window 45/289 failed (KQL: let _startTime = datetime(2026-10-15T01:14:38.7716298Z);\nlet _endTime = datetime(2026-10-15T01:19:38.7716297Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled",
      "name": "backendControllerConditions"
    }�[0m
    �[37m[21:34:47.129]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{
      "input": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/custom/custom-query_cosmosResourceSnapshots.jsonl",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/cosmosContent"
    }�[0m
    �[37m[21:34:47.129]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/custom/custom-query_cosmosResourceSnapshots.jsonl"
    }�[0m
    �[37m[21:34:47.633]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{
      "count": 1498,
      "files": 1
    }�[0m
    �[37m[21:34:47.633]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/cosmosContent"
    }�[0m
    �[37m[21:34:47.642]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m
    �[37m[21:34:59.291]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{
      "clusterIDs": null,
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect"
    }�[0m
    �[37m[21:34:59.911]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{
      "clusterIDs": [
        "2tfevok49t0kjg6c48lr24m7jgn4lk62"
      ]
    }�[0m
    �[37m[21:35:00.221]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{
      "clusterNames": [
        "ci01-j8700672-mgmt-1",
        "ci01-j8700672-svc"
      ]
    }�[0m
    �[37m[21:35:00.360]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespaces": [
        "2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g"
      ]
    }�[0m
    �[37m[21:35:00.360]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{
      "managementCluster": "ci01-j8700672-mgmt-1",
      "namespaceCount": 4
    }�[0m
    �[37m[21:35:00.360]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:35:00.702]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:35:01.111]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:35:01.638]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g"
    }�[0m
    �[37m[21:35:02.339]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:34:46.3046137Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\", ActivityId:\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\", SubActivityId:\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\", ActivityStack:\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\"}, IsPermanent:false})\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\n",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904"
    }�[0m
    �[37m[21:35:02.340]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:34:46.3046137Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\", ActivityId:\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\", SubActivityId:\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\", ActivityStack:\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\"}, IsPermanent:false})\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\n"
    }�[0m
    
      query-skip-hcp-and-events: command failed: exit status 2
    output: �[37m[21:35:02.599]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-1950984811"
    }�[0m
    runtime/cgo: pthread_create failed: Resource temporarily unavailable
    SIGABRT: abort
    PC=0x7f80df60594c m=4 sigcode=18446744073709551610
    
    goroutine 0 gp=0x14197f1334a0 m=4 mp=0x14197f21a008 [idle]:
    runtime: g 0 gp=0x14197f1334a0: unknown pc 0x7f80df60594c
    stack: frame={sp:0x7f808bffe790, fp:0x0} stack=[0x7f808b7ff1e0,0x7f808bffede0)
                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef
    00007f808bffe690: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6a0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6b0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6c0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6d0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6e0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6f0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe700: 00000000 00000000  00000000 00000000  ................
    00007f808bffe710: 00000000 00000000  00000000 00000000  ................
    00007f808bffe720: 00000000 00000000  00000000 00000000  ................
    00007f808bffe730: 00000000 00000000  00000000 00000000  ................
    00007f808bffe740: 00000000 00000000  00000000 00000000  ................
    00007f808bffe750: 00000000 00000000  00000000 00000000  ................
    00007f808bffe760: 00000000 00000000  00007f80 df612fa9  ........./a.....
    00007f808bffe770: 00000000 00000000  00000000 00000011  ................
    00007f808bffe780: 00000000 00000130  00007f80 df60593e  0.......>Y`.....
    00007f808bffe790: 00000000 00000120  00000000 00000000   ...............
                      ^ SP
    00007f808bffe7a0: 00010000 00000013  00000000 00000000  ................
    00007f808bffe7b0: 00000000 00000000  00007f80 df688cff  ..........h.....
    00007f808bffe7c0: 00007f80 89ffb910  00007f80 df688cb1  ..........h.....
    00007f808bffe7d0: 00007f80 89ffb640  00007f80 8bffeaa0  @...............
    00007f808bffe7e0: 00007f80 8bffe90e  00007f80 8bffe90f  ................
    00007f808bffe7f0: 00007f80 89ffb640  00007f80 df6037f5  @........7`.....
    00007f808bffe800: 00007f80 84000020  00000000 00000120   ....... .......
    00007f808bffe810: 00000000 003d0f00  701e1ebb c8982900  ..=......).....p
    00007f808bffe820: 00007f80 8bfff640  00000000 00000006  @...............
    00007f808bffe830: 00007f80 84000c90  00007f80 8bffeb60  ........`.......
    00007f808bffe840: 00001419 7f1334a0  00007f80 df5b8646  .4......F.[.....
    00007f808bffe850: 00007f80 df775e90  00007f80 df5a27f3  .^w......'Z.....
    00007f808bffe860: 00000000 00000020  701e1ebb c8982900   ........).....p
    00007f808bffe870: 00007f80 897fb000  00007f80 df677ac4  .........zg.....
    00007f808bffe880: 00000000 00000000  00000000 0000000d  ................
    runtime: g 0 gp=0x14197f1334a0: unknown pc 0x7f80df60594c
    stack: frame={sp:0x7f808bffe790, fp:0x0} stack=[0x7f808b7ff1e0,0x7f808bffede0)
                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef
    00007f808bffe690: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6a0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6b0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6c0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6d0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6e0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6f0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe700: 00000000 00000000  00000000 00000000  ................
    00007f808bffe710: 00000000 00000000  00000000 00000000  ................
    00007f808bffe720: 00000000 00000000  00000000 00000000  ................
    00007f808bffe730: 00000000 00000000  00000000 00000000  ................
    00007f808bffe740: 00000000 00000000  00000000 00000000  ................
    00007f808bffe750: 00000000 00000000  00000000 00000000  ................
    00007f808bffe760: 00000000 00000000  00007f80 df612fa9  ........./a.....
    00007f808bffe770: 00000000 00000000  00000000 00000011  ................
    00007f808bffe780: 00000000 00000130  00007f80 df60593e  0.......>Y`.....
    00007f808bffe790: 00000000 00000120  00000000 00000000   ...............
                      ^ SP
    00007f808bffe7a0: 00010000 00000013  00000000 00000000  ................
    00007f808bffe7b0: 00000000 00000000  00007f80 df688cff  ..........h.....
    00007f808bffe7c0: 00007f80 89ffb910  00007f80 df688cb1  ..........h.....
    00007f808bffe7d0: 00007f80 89ffb640  00007f80 8bffeaa0  @...............
    00007f808bffe7e0: 00007f80 8bffe90e  00007f80 8bffe90f  ................
    00007f808bffe7f0: 00007f80 89ffb640  00007f80 df6037f5  @........7`.....
    00007f808bffe800: 00007f80 84000020  00000000 00000120   ....... .......
    00007f808bffe810: 00000000 003d0f00  701e1ebb c8982900  ..=......).....p
    00007f808bffe820: 00007f80 8bfff640  00000000 00000006  @...............
    00007f808bffe830: 00007f80 84000c90  00007f80 8bffeb60  ........`.......
    00007f808bffe840: 00001419 7f1334a0  00007f80 df5b8646  .4......F.[.....
    00007f808bffe850: 00007f80 df775e90  00007f80 df5a27f3  .^w......'Z.....
    00007f808bffe860: 00000000 00000020  701e1ebb c8982900   ........).....p
    00007f808bffe870: 00007f80 897fb000  00007f80 df677ac4  .........zg.....
    00007f808bffe880: 00000000 00000000  00000000 0000000d  ................
    
    goroutine 1 gp=0x14197f1321e0 m=nil [sync.WaitGroup.Wait]:
    runtime.gopark(0x14197fe86c00?, 0x2022a4d?, 0x10?, 0x63?, 0x14197fe86c30?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197ff32be8 sp=0x14197ff32bc8 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x14197fc98610, 0x0, 0x1, 0x0, 0x19)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197ff32c50 sp=0x14197ff32be8 pc=0x1f162b2
    sync.runtime_SemacquireWaitGroup(0x14197fd289c0?, 0xe0?)
    	/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x14197ff32c88 sp=0x14197ff32c50 pc=0x1f3a66e
    sync.(*WaitGroup).Wait(0x14197fc98608)
    	/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x14197ff32cb0 sp=0x14197ff32c88 pc=0x1f4f445
    golang.org/x/sync/errgroup.(*Group).Wait(0x14197fc98600)
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x14197ff32cd0 sp=0x14197ff32cb0 pc=0x362507e
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x14197f521dc0, {0x59f0b08, 0x14197f530fc0}, {0x14197fcca000, 0x15, 0x1c})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x14197ff32d90 sp=0x14197ff32cd0 pc=0x3927e05
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x14197f521dc0, {0x59f0b08, 0x14197f530fc0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:500 +0x4b08 fp=0x14197ff33968 sp=0x14197ff32d90 pc=0x39261c8
    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedQueryOptions).RunQuery(0x14197f1401e0, {0x59f0b08, 0x14197f530fc0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_cmd.go:75 +0x565 fp=0x14197ff33b30 sp=0x14197ff33968 pc=0x3937005
    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawQueryOptions).Run(0x0?, {0x59f0b08, 0x14197f530fc0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_options.go:178 +0x54 fp=0x14197ff33b58 sp=0x14197ff33b30 pc=0x3939d34
    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryCommand.func1(0x14197f4ba008?, {0x14197f5f82a0?, 0x4?, 0x553b0e2?})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_cmd.go:45 +0x28 fp=0x14197ff33b80 sp=0x14197ff33b58 pc=0x3936a68
    github.com/spf13/cobra.(*Command).execute(0x14197f4ba008, {0x14197f2c0030, 0xe, 0xf})
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x14197ff33d28 sp=0x14197ff33b80 pc=0x212e614
    github.com/spf13/cobra.(*Command).ExecuteC(0x14197f625b08)
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x14197ff33e18 sp=0x14197ff33d28 pc=0x212efa5
    github.com/spf13/cobra.(*Command).Execute(...)
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071
    github.com/spf13/cobra.(*Command).ExecuteContext(...)
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064
    main.main()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x14197ff33f48 sp=0x14197ff33e18 pc=0x47ddd6a
    runtime.main()
    	/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x14197ff33fe0 sp=0x14197ff33f48 pc=0x1f01975
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197ff33fe8 sp=0x14197ff33fe0 pc=0x1f40801
    
    goroutine 2 gp=0x14197f132d20 m=nil [force gc (idle)]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f2fa8 sp=0x14197f1f2f88 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.forcegchelper()
    	/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x14197f1f2fe0 sp=0x14197f1f2fa8 pc=0x1f01c93
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f2fe8 sp=0x14197f1f2fe0 pc=0x1f40801
    created by runtime.init.7 in goroutine 1
    	/usr/lib/golang/src/runtime/proc.go:363 +0x1a
    
    goroutine 18 gp=0x14197f2843c0 m=nil [GC sweep wait]:
    runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1ee788 sp=0x14197f1ee768 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.bgsweep(0x14197f292000)
    	/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x14197f1ee7c8 sp=0x14197f1ee788 pc=0x1ee9f11
    runtime.gcenable.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x14197f1ee7e0 sp=0x14197f1ee7c8 pc=0x1edadd7
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1ee7e8 sp=0x14197f1ee7e0 pc=0x1f40801
    created by runtime.gcenable in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:214 +0x66
    
    goroutine 19 gp=0x14197f2845a0 m=nil [GC scavenge wait]:
    runtime.gopark(0x138d6d?, 0x3979fd?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1eef78 sp=0x14197f1eef58 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.(*scavengerState).park(0x82979c0)
    	/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x14197f1eefa8 sp=0x14197f1eef78 pc=0x1ee77e9
    runtime.bgscavenge(0x14197f292000)
    	/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x14197f1eefc8 sp=0x14197f1eefa8 pc=0x1ee7d99
    runtime.gcenable.gowrap2()
    	/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x14197f1eefe0 sp=0x14197f1eefc8 pc=0x1edad97
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1eefe8 sp=0x14197f1eefe0 pc=0x1f40801
    created by runtime.gcenable in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:215 +0xa5
    
    goroutine 20 gp=0x14197f284780 m=nil [GOMAXPROCS updater (idle)]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1ef788 sp=0x14197f1ef768 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.updateMaxProcsGoroutine()
    	/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x14197f1ef7e0 sp=0x14197f1ef788 pc=0x1f10267
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1ef7e8 sp=0x14197f1ef7e0 pc=0x1f40801
    created by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1
    	/usr/lib/golang/src/runtime/proc.go:7083 +0x37
    
    goroutine 21 gp=0x14197f284d20 m=nil [finalizer wait]:
    runtime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x14197f1f2601?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f2620 sp=0x14197f1f2600 pc=0x1f38d2e
    runtime.runFinalizers()
    	/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x14197f1f27e0 sp=0x14197f1f2620 pc=0x1ed9cc7
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f27e8 sp=0x14197f1f27e0 pc=0x1f40801
    created by runtime.createfing in goroutine 1
    	/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d
    
    goroutine 22 gp=0x14197f47c960 m=nil [cleanup wait]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1eff68 sp=0x14197f1eff48 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.(*cleanupQueue).dequeue(0x82980e0)
    	/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x14197f1effa0 sp=0x14197f1eff68 pc=0x1ed6074
    runtime.runCleanups()
    	/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x14197f1effe0 sp=0x14197f1effa0 pc=0x1ed66e5
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1effe8 sp=0x14197f1effe0 pc=0x1f40801
    created by runtime.(*cleanupQueue).createGs in goroutine 1
    	/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5
    
    goroutine 23 gp=0x14197f5a30e0 m=nil [GC worker (idle)]:
    runtime.gopark(0x125b862b4ec7?, 0x1?, 0x21?, 0xab?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f0740 sp=0x14197f1f0720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x14197f29d9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f07c8 sp=0x14197f1f0740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f07e0 sp=0x14197f1f07c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f07e8 sp=0x14197f1f07e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 3 gp=0x14197f133680 m=nil [runnable]:
    runtime.gopark(0x125b8625d223?, 0x3?, 0x89?, 0x19?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f3740 sp=0x14197f1f3720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x14197f29d9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f37c8 sp=0x14197f1f3740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f37e0 sp=0x14197f1f37c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f37e8 sp=0x14197f1f37e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 34 gp=0x14197f604000 m=nil [GC worker (idle)]:
    runtime.gopark(0x125b862e8e02?, 0x3?, 0xe6?, 0x2d?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60a740 sp=0x14197f60a720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x14197f29d9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f60a7c8 sp=0x14197f60a740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f60a7e0 sp=0x14197f60a7c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60a7e8 sp=0x14197f60a7e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 24 gp=0x14197f5a32c0 m=nil [GC worker (idle)]:
    runtime.gopark(0x125b862118da?, 0x1?, 0x4d?, 0x9a?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f0f40 sp=0x14197f1f0f20 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x14197f29d9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f0fc8 sp=0x14197f1f0f40 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f0fe0 sp=0x14197f1f0fc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f0fe8 sp=0x14197f1f0fe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 4 gp=0x14197f133860 m=nil [GC worker (idle)]:
    runtime.gopark(0x125b7fe90140?, 0x3?, 0x20?, 0x3a?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f3f40 sp=0x14197f1f3f20 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x14197f29d9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f3fc8 sp=0x14197f1f3f40 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f3fe0 sp=0x14197f1f3fc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f3fe8 sp=0x14197f1f3fe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 35 gp=0x14197f6041e0 m=nil [GC worker (idle)]:
    runtime.gopark(0x125b862a34bd?, 0x3?, 0x4?, 0x15?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60af40 sp=0x14197f60af20 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x14197f29d9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f60afc8 sp=0x14197f60af40 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f60afe0 sp=0x14197f60afc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60afe8 sp=0x14197f60afe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 25 gp=0x14197f5a34a0 m=3 mp=0x14197f1a5808 [GC mark termination]:
    runtime.systemstack_switch()
    	/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x14197f1f16a0 sp=0x14197f1f1690 pc=0x1f3ebc8
    runtime.forEachP(...)
    	/usr/lib/golang/src/runtime/proc.go:2112
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x14197f1f1740 sp=0x14197f1f16a0 pc=0x1edbc34
    runtime.gcBgMarkWorker(0x14197f29d9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x14197f1f17c8 sp=0x14197f1f1740 pc=0x1edda88
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f17e0 sp=0x14197f1f17c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f17e8 sp=0x14197f1f17e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 5 gp=0x14197f133a40 m=nil [GC worker (idle)]:
    runtime.gopark(0x125b8624ea9b?, 0x1?, 0xfd?, 0x7a?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f4740 sp=0x14197f1f4720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x14197f29d9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f47c8 sp=0x14197f1f4740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f47e0 sp=0x14197f1f47c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f47e8 sp=0x14197f1f47e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 50 gp=0x14197f768f00 m=nil [select, locked to thread]:
    runtime.gopark(0x14197f60c7a8?, 0x2?, 0x1?, 0x8f?, 0x14197f60c794?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60c628 sp=0x14197f60c608 pc=0x1f38d2e
    runtime.selectgo(0x14197f60c7a8, 0x14197f60c790, 0x0?, 0x0, 0x0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f60c758 sp=0x14197f60c628 pc=0x1f152a5
    runtime.ensureSigM.func1()
    	/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x14197f60c7e0 sp=0x14197f60c758 pc=0x1f328e8
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60c7e8 sp=0x14197f60c7e0 pc=0x1f40801
    created by runtime.ensureSigM in goroutine 1
    	/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5
    
    goroutine 26 gp=0x14197f3b34a0 m=7 mp=0x14197f55b008 [syscall]:
    runtime.notetsleepg(0x82bdea0, 0xffffffffffffffff)
    	/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x14197f6077a0 sp=0x14197f607778 pc=0x1ece509
    os/signal.signal_recv()
    	/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x14197f6077c0 sp=0x14197f6077a0 pc=0x1f3b298
    os/signal.loop()
    	/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x14197f6077e0 sp=0x14197f6077c0 pc=0x2030553
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f6077e8 sp=0x14197f6077e0 pc=0x1f40801
    created by os/signal.Notify.func1.1 in goroutine 1
    	/usr/lib/golang/src/os/signal/signal.go:152 +0x1f
    
    goroutine 27 gp=0x14197f6894a0 m=nil [select]:
    runtime.gopark(0x14197f607f98?, 0x2?, 0x68?, 0x81?, 0x14197f607f8c?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f607e28 sp=0x14197f607e08 pc=0x1f38d2e
    runtime.selectgo(0x14197f607f98, 0x14197f607f88, 0x0?, 0x0, 0x0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f607f58 sp=0x14197f607e28 pc=0x1f152a5
    os/signal.NotifyContext.func1()
    	/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x14197f607fe0 sp=0x14197f607f58 pc=0x202ff9c
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f607fe8 sp=0x14197f607fe0 pc=0x1f40801
    created by os/signal.NotifyContext in goroutine 1
    	/usr/lib/golang/src/os/signal/signal.go:291 +0x158
    
    goroutine 224 gp=0x14197f689680 m=nil [semacquire]:
    runtime.gopark(0x7f80904adba8?, 0x1eed7e0?, 0xc0?, 0x68?, 0x4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419800b5b58 sp=0x1419800b5b38 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x1419800b5bc0 sp=0x1419800b5b58 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x1419800b5c60 sp=0x1419800b5bc0 pc=0x1edbb12
    runtime.gcAssistAlloc(0x14197f689680)
    	/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x1419800b5cf0 sp=0x1419800b5c60 pc=0x1ee06b6
    runtime.deductAssistCredit(0x7ffce51f9080?)
    	/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x1419800b5d10 sp=0x1419800b5cf0 pc=0x1ed10a6
    runtime.mallocgc(0x7fe00, 0x4af32a0, 0x1)
    	/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x1419800b5d40 sp=0x1419800b5d10 pc=0x1f35e07
    runtime.makeslice(0x14197fc9d0b0?, 0x141983a1fe00?, 0x20000?)
    	/usr/lib/golang/src/runtime/slice.go:117 +0x49 fp=0x1419800b5d68 sp=0x1419800b5d40 pc=0x1f3b489
    encoding/json.(*Decoder).refill(0x14197f658c80)
    	/usr/lib/golang/src/encoding/json/stream.go:161 +0xee fp=0x1419800b5db8 sp=0x1419800b5d68 pc=0x200d48e
    encoding/json.(*Decoder).readValue(0x14197f658c80)
    	/usr/lib/golang/src/encoding/json/stream.go:142 +0x85 fp=0x1419800b5e08 sp=0x1419800b5db8 pc=0x200d125
    encoding/json.(*Decoder).Decode(0x14197f658c80, {0x4cfb340, 0x14197ffa44c0})
    	/usr/lib/golang/src/encoding/json/stream.go:65 +0x75 fp=0x1419800b5e38 sp=0x1419800b5e08 pc=0x200cdf5
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readPrimaryTable(0x14197f8c54f0, 0x14197fb50000)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:264 +0x16e fp=0x1419800b5ec8 sp=0x1419800b5e38 pc=0x38d822e
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x14197f8c54f0)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:174 +0x189 fp=0x1419800b5f30 sp=0x1419800b5ec8 pc=0x38d7b89
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x14197f8c54f0, 0x14198007be90)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x1419800b5fc0 sp=0x1419800b5f30 pc=0x38d788c
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x1419800b5fe0 sp=0x1419800b5fc0 pc=0x38d751b
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1419800b5fe8 sp=0x1419800b5fe0 pc=0x1f40801
    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 183
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc
    
    goroutine 87 gp=0x14197fd3c000 m=nil [sync.WaitGroup.Wait]:
    runtime.gopark(0x14197fcf9e00?, 0x1ed0734?, 0x20?, 0x94?, 0x7f80917bdf06?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf9da0 sp=0x14197fcf9d80 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x14197f69cc50, 0x0, 0x1, 0x0, 0x19)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fcf9e08 sp=0x14197fcf9da0 pc=0x1f162b2
    sync.runtime_SemacquireWaitGroup(0x14197f86b2d8?, 0x0?)
    	/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x14197fcf9e40 sp=0x14197fcf9e08 pc=0x1f3a66e
    sync.(*WaitGroup).Wait(0x14197f69cc48)
    	/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x14197fcf9e68 sp=0x14197fcf9e40 pc=0x1f4f445
    golang.org/x/sync/errgroup.(*Group).Wait(0x14197f69cc40)
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x14197fcf9e88 sp=0x14197fcf9e68 pc=0x362507e
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x14197f521d80, {0x59f0b40, 0x14197f8c4b40}, {0x14197fcca000, 0x15, 0x6f63746669687372?}, 0x14197f1c02a0)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x14197fcf9f00 sp=0x14197fcf9e88 pc=0x392976a
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x14197fcf9f78 sp=0x14197fcf9f00 pc=0x392808e
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197fcf9fe0 sp=0x14197fcf9f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf9fe8 sp=0x14197fcf9fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 77 gp=0x14197fd3c1e0 m=nil [IO wait]:
    runtime.gopark(0x14197fb3e4a0?, 0x1edb176?, 0x80?, 0xed?, 0xb?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f202808 sp=0x14197f2027e8 pc=0x1f38d2e
    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)
    	/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x14197f202840 sp=0x14197f202808 pc=0x1ef9e57
    internal/poll.runtime_pollWait(0x7f8098109000, 0x72)
    	/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x14197f202860 sp=0x14197f202840 pc=0x1f375c5
    internal/poll.(*pollDesc).wait(0x14197f323a80?, 0x14197f34d300?, 0x0)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x14197f202888 sp=0x14197f202860 pc=0x1fbd167
    internal/poll.(*pollDesc).waitRead(...)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89
    internal/poll.(*FD).Read(0x14197f323a80, {0x14197f34d300, 0x1300, 0x1300})
    	/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x14197f202920 sp=0x14197f202888 pc=0x1fbe4ce
    net.(*netFD).Read(0x14197f323a80, {0x14197f34d300?, 0x14197f34d30a?, 0x5?})
    	/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x14197f202968 sp=0x14197f202920 pc=0x20798c5
    net.(*conn).Read(0x14197f1300d8, {0x14197f34d300?, 0x7f809057a868?, 0x7f80df53a6d8?})
    	/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x14197f2029b0 sp=0x14197f202968 pc=0x208b7a5
    crypto/tls.(*atLeastReader).Read(0x14197f821a10, {0x14197f34d300?, 0x14197f202a98?, 0x1efcc28?})
    	/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x14197f2029f8 sp=0x14197f2029b0 pc=0x2222ddb
    bytes.(*Buffer).ReadFrom(0x14197f8a57a8, {0x59a3680, 0x14197f821a10})
    	/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x14197f202a50 sp=0x14197f2029f8 pc=0x1fe9258
    crypto/tls.(*Conn).readFromUntil(0x14197f8a5508, {0x59a32c0, 0x14197f1300d8}, 0x222764c?)
    	/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x14197f202a88 sp=0x14197f202a50 pc=0x2222fbe
    crypto/tls.(*Conn).readRecordOrCCS(0x14197f8a5508, 0x0)
    	/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x14197f202c68 sp=0x14197f202a88 pc=0x222009b
    crypto/tls.(*Conn).readRecord(...)
    	/usr/lib/golang/src/crypto/tls/conn.go:588
    crypto/tls.(*Conn).Read(0x14197f8a5508, {0x14197fa26000, 0x1000, 0x4e55540?})
    	/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x14197f202cd0 sp=0x14197f202c68 pc=0x22267c5
    net/http.(*persistConn).Read(0x14197f8032c0, {0x14197fa26000?, 0x59a0ea0?, 0x81260e0?})
    	/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x14197f202d30 sp=0x14197f202cd0 pc=0x2319d07
    bufio.(*Reader).fill(0x14197f29b620)
    	/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f202d68 sp=0x14197f202d30 pc=0x20a6c63
    bufio.(*Reader).Peek(0x14197f29b620, 0x1)
    	/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x14197f202d80 sp=0x14197f202d68 pc=0x20a6df3
    net/http.(*persistConn).readLoop(0x14197f8032c0)
    	/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x14197f202fc8 sp=0x14197f202d80 pc=0x231a812
    net/http.(*Transport).dialConn.gowrap2()
    	/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x14197f202fe0 sp=0x14197f202fc8 pc=0x2319197
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f202fe8 sp=0x14197f202fe0 pc=0x1f40801
    created by net/http.(*Transport).dialConn in goroutine 9
    	/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac
    
    goroutine 7 gp=0x14197fe025a0 m=nil [sync.Mutex.Lock]:
    runtime.gopark(0x14197f9d3c68?, 0x14197f9d3c70?, 0x94?, 0x69?, 0x14197f9d3c78?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f9d3c00 sp=0x14197f9d3be0 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x14197fccaad4, 0x0, 0x3, 0x2, 0x16)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197f9d3c68 sp=0x14197f9d3c00 pc=0x1f162b2
    internal/sync.runtime_SemacquireMutex(0x1000?, 0xd0?, 0x1ed11a5?)
    	/usr/lib/golang/src/runtime/sema.go:95 +0x25 fp=0x14197f9d3ca0 sp=0x14197f9d3c68 pc=0x1f3a545
    internal/sync.(*Mutex).lockSlow(0x14197fccaad0)
    	/usr/lib/golang/src/internal/sync/mutex.go:149 +0x15d fp=0x14197f9d3cf0 sp=0x14197f9d3ca0 pc=0x1f4d31d
    internal/sync.(*Mutex).Lock(...)
    	/usr/lib/golang/src/internal/sync/mutex.go:70
    sync.(*Mutex).Lock(...)
    	/usr/lib/golang/src/sync/mutex.go:46
    net/http.(*http2clientConnReadLoop).streamByID(0x22babb5?, 0x1b, 0x1)
    	/usr/lib/golang/src/net/http/h2_bundle.go:10083 +0x5b fp=0x14197f9d3d48 sp=0x14197f9d3cf0 pc=0x22e10db
    net/http.(*http2clientConnReadLoop).processData(0x14197f9d3fa8, 0x14197fc9d200)
    	/usr/lib/golang/src/net/http/h2_bundle.go:9939 +0x45 fp=0x14197f9d3e18 sp=0x14197f9d3d48 pc=0x22e05e5
    net/http.(*http2clientConnReadLoop).run(0x14197f9d3fa8)
    	/usr/lib/golang/src/net/http/h2_bundle.go:9584 +0x4b6 fp=0x14197f9d3f60 sp=0x14197f9d3e18 pc=0x22de536
    net/http.(*http2ClientConn).readLoop(0x14197fccaa80)
    	/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x14197f9d3fc8 sp=0x14197f9d3f60 pc=0x22dd532
    net/http.(*http2Transport).newClientConn.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x14197f9d3fe0 sp=0x14197f9d3fc8 pc=0x22d7057
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f9d3fe8 sp=0x14197f9d3fe0 pc=0x1f40801
    created by net/http.(*http2Transport).newClientConn in goroutine 6
    	/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5
    
    goroutine 78 gp=0x14197fe02960 m=nil [select]:
    runtime.gopark(0x14197f9f7f48?, 0x2?, 0x0?, 0xb6?, 0x14197f9f7ef4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf2d88 sp=0x14197fcf2d68 pc=0x1f38d2e
    runtime.selectgo(0x14197fcf2f48, 0x14197f9f7ef0, 0x14197f1cd0c0?, 0x0, 0x14197f89c2a0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197fcf2eb8 sp=0x14197fcf2d88 pc=0x1f152a5
    net/http.(*persistConn).writeLoop(0x14197f8032c0)
    	/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x14197fcf2fc8 sp=0x14197fcf2eb8 pc=0x231c426
    net/http.(*Transport).dialConn.gowrap3()
    	/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x14197fcf2fe0 sp=0x14197fcf2fc8 pc=0x2319157
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf2fe8 sp=0x14197fcf2fe0 pc=0x1f40801
    created by net/http.(*Transport).dialConn in goroutine 9
    	/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe
    
    goroutine 163 gp=0x14197f7694a0 m=nil [select]:
    runtime.gopark(0x14197fdd3f08?, 0x6?, 0xa0?, 0x94?, 0x14197fdd3da4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f205c08 sp=0x14197f205be8 pc=0x1f38d2e
    runtime.selectgo(0x14197f205f08, 0x14197fdd3d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f205d38 sp=0x14197f205c08 pc=0x1f152a5
    net/http.(*http2clientStream).writeRequest(0x14197f37f380, 0x14197f49e140, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x14197f205f90 sp=0x14197f205d38 pc=0x22da6a8
    net/http.(*http2clientStream).doRequest(0x14197f37f380, 0x14197f546100?, 0x14197f609798?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x14197f205fb8 sp=0x14197f205f90 pc=0x22d99d8
    net/http.(*http2ClientConn).roundTrip.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197f205fe0 sp=0x14197f205fb8 pc=0x22d999f
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f205fe8 sp=0x14197f205fe0 pc=0x1f40801
    created by net/http.(*http2ClientConn).roundTrip in goroutine 184
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470
    
    goroutine 88 gp=0x14197f769a40 m=nil [semacquire]:
    runtime.gopark(0x141980119958?, 0x1ed3e05?, 0xe8?, 0xd2?, 0x7f80904ad2e8?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x141980119908 sp=0x1419801198e8 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x141980119970 sp=0x141980119908 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x141980119a10 sp=0x141980119970 pc=0x1edbb12
    runtime.gcAssistAlloc(0x14197f769a40)
    	/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x141980119aa0 sp=0x141980119a10 pc=0x1ee06b6
    runtime.deductAssistCredit(0x141983b00000?)
    	/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x141980119ac0 sp=0x141980119aa0 pc=0x1ed10a6
    runtime.mallocgc(0x7edd, 0x0, 0x0)
    	/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x141980119af0 sp=0x141980119ac0 pc=0x1f35e07
    runtime.slicebytetostring(0x4e537a0?, 0x141983b00000, 0x7edd)
    	/usr/lib/golang/src/runtime/string.go:170 +0x77 fp=0x141980119b20 sp=0x141980119af0 pc=0x1f1e397
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x14197f8c4b40}, 0x14197f1c03f0, 0x14197f531740)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:322 +0x91b fp=0x141980119f48 sp=0x141980119b20 pc=0x3920bfb
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x141980119f78 sp=0x141980119f48 pc=0x392800c
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x141980119fe0 sp=0x141980119f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x141980119fe8 sp=0x141980119fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 110 gp=0x14197fd3cb40 m=nil [sync.Mutex.Lock]:
    runtime.gopark(0x22b8a73?, 0x14197f7e1000?, 0xb0?, 0x4c?, 0x8160a40?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1c7510 sp=0x14197f1c74f0 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x14197fccaad4, 0x0, 0x3, 0x2, 0x16)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197f1c7578 sp=0x14197f1c7510 pc=0x1f162b2
    internal/sync.runtime_SemacquireMutex(0x14197f1c75d0?, 0x66?, 0x61f?)
    	/usr/lib/golang/src/runtime/sema.go:95 +0x25 fp=0x14197f1c75b0 sp=0x14197f1c7578 pc=0x1f3a545
    internal/sync.(*Mutex).lockSlow(0x14197fccaad0)
    	/usr/lib/golang/src/internal/sync/mutex.go:149 +0x15d fp=0x14197f1c7600 sp=0x14197f1c75b0 pc=0x1f4d31d
    internal/sync.(*Mutex).Lock(...)
    	/usr/lib/golang/src/internal/sync/mutex.go:70
    sync.(*Mutex).Lock(...)
    	/usr/lib/golang/src/sync/mutex.go:46
    net/http.http2transportResponseBody.Read({0x1f3ec12?}, {0x14198313d000?, 0x1eed797?, 0x14197f1c76b0?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:9875 +0x209 fp=0x14197f1c7688 sp=0x14197f1c7600 pc=0x22dff89
    bufio.(*Reader).fill(0x14197f3a00c0)
    	/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f1c76c0 sp=0x14197f1c7688 pc=0x20a6c63
    bufio.(*Reader).ReadByte(0x14197f3a00c0)
    	/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x14197f1c76d8 sp=0x14197f1c76c0 pc=0x20a73c7
    compress/flate.(*decompressor).moreBits(0x141980059908)
    	/usr/lib/golang/src/compress/flate/inflate.go:697 +0x24 fp=0x14197f1c7710 sp=0x14197f1c76d8 pc=0x2288824
    compress/flate.(*decompressor).nextBlock(0x141980059908)
    	/usr/lib/golang/src/compress/flate/inflate.go:304 +0x28 fp=0x14197f1c7728 sp=0x14197f1c7710 pc=0x2286ba8
    compress/flate.(*decompressor).Read(0x141980059908, {0x14198314e000, 0x1000, 0x7f8090ef0f00?})
    	/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x14197f1c7758 sp=0x14197f1c7728 pc=0x2286dbb
    compress/gzip.(*Reader).Read(0x14197f672dc8, {0x14198314e000, 0x1000, 0x1000})
    	/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x14197f1c77e8 sp=0x14197f1c7758 pc=0x228c838
    github.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x410d0fa76c5d7d40?, {0x14198314e000?, 0x7f8090ef0d20?, 0x52ffb40?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x14197f1c7818 sp=0x14197f1c77e8 pc=0x38b33e2
    bufio.(*Reader).fill(0x14197f3a0120)
    	/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f1c7850 sp=0x14197f1c7818 pc=0x20a6c63
    bufio.(*Reader).Peek(0x14197f3a0120, 0x1)
    	/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x14197f1c7868 sp=0x14197f1c7850 pc=0x20a6df3
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.validateJsonResponse(0x14197f3a0120)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:38 +0x25 fp=0x14197f1c78b8 sp=0x14197f1c7868 pc=0x38d6be5
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.newFrameReader({0x59db110, 0x14197fe1e040}, {0x59f0b40, 0x14197f748140})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:26 +0x188 fp=0x14197f1c7960 sp=0x14197f1c78b8 pc=0x38d6a08
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset({0x59f0bb0?, 0x14197f4797a0?}, {0x59db110, 0x14197fe1e040}, 0x1, 0x3e8, 0x1)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:66 +0x19e fp=0x14197f1c79e0 sp=0x14197f1c7960 pc=0x38d737e
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x14197f521d40, {0x59f0bb0, 0x14197f4797a0}, {0x14197f49a330, 0xb}, 0x14197fe1e700, {0x0?, 0x2?, 0x0?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:167 +0x2b7 fp=0x14197f1c7a78 sp=0x14197f1c79e0 pc=0x38e4f17
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197f7a2600}, 0x14197f1c02a0)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x14197f1c7d60 sp=0x14197f1c7a78 pc=0x391b5c5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x14197f521d80, {0x59f0b40, 0x14197fdd8140}, {0x5a00340, 0x14197fc1e5d0}, 0x14197f1c02a0)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:98 +0x366 fp=0x14197f1c7e88 sp=0x14197f1c7d60 pc=0x3929ec6
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14197f1c7f78 sp=0x14197f1c7e88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197f1c7fe0 sp=0x14197f1c7f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1c7fe8 sp=0x14197f1c7fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 89 gp=0x14197fd82960 m=nil [select]:
    runtime.gopark(0x1419800b7d00?, 0x2?, 0xd7?, 0xfd?, 0x1419800b7c74?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419800b7ac8 sp=0x1419800b7aa8 pc=0x1f38d2e
    runtime.selectgo(0x1419800b7d00, 0x1419800b7c70, 0x1419801521c0?, 0x0, 0x14197f5daba0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x1419800b7bf8 sp=0x1419800b7ac8 pc=0x1f152a5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x14197f521d80?, {0x59f0b40, 0x14197f8c4b40}, 0x14197f1c02a0, 0x14197f1c03f0)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x1419800b7f10 sp=0x1419800b7bf8 pc=0x392836e
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x1419800b7f78 sp=0x1419800b7f10 pc=0x3927f3d
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x1419800b7fe0 sp=0x1419800b7f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1419800b7fe8 sp=0x1419800b7fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 175 gp=0x14197fd3d2c0 m=nil [sync.Cond.Wait]:
    runtime.gopark(0x22c2d5b?, 0x14197fe1c080?, 0x0?, 0xe0?, 0x14197f777000?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14198012b978 sp=0x14198012b958 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    sync.runtime_notifyListWait(0x14197f37f3c8, 0x3c)
    	/usr/lib/golang/src/runtime/sema.go:617 +0x1b3 fp=0x14198012b9c8 sp=0x14198012b978 pc=0x1f3a8f3
    sync.(*Cond).Wait(0x1000?)
    	/usr/lib/golang/src/sync/cond.go:71 +0x73 fp=0x14198012ba00 sp=0x14198012b9c8 pc=0x1f4d673
    net/http.(*http2pipe).Read(0x14197f37f3b0, {0x14197ff7e000, 0x1000, 0x1000})
    	/usr/lib/golang/src/net/http/h2_bundle.go:3928 +0xdb fp=0x14198012ba68 sp=0x14198012ba00 pc=0x22c2c1b
    net/http.http2transportResponseBody.Read({0x4a54?}, {0x14197ff7e000?, 0x14198012bb80?, 0x2287949?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:9852 +0x59 fp=0x14198012baf0 sp=0x14198012ba68 pc=0x22dfdd9
    bufio.(*Reader).fill(0x14197fcaa000)
    	/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14198012bb28 sp=0x14198012baf0 pc=0x20a6c63
    bufio.(*Reader).ReadByte(0x14197fcaa000)
    	/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x14198012bb40 sp=0x14198012bb28 pc=0x20a73c7
    compress/flate.(*decompressor).moreBits(0x141980056008)
    	/usr/lib/golang/src/compress/flate/inflate.go:697 +0x24 fp=0x14198012bb78 sp=0x14198012bb40 pc=0x2288824
    compress/flate.(*decompressor).nextBlock(0x141980056008)
    	/usr/lib/golang/src/compress/flate/inflate.go:304 +0x28 fp=0x14198012bb90 sp=0x14198012bb78 pc=0x2286ba8
    compress/flate.(*decompressor).Read(0x141980056008, {0x141980034b13, 0x4ed, 0xb13?})
    	/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x14198012bbc0 sp=0x14198012bb90 pc=0x2286dbb
    compress/gzip.(*Reader).Read(0x141980000b08, {0x141980034b13, 0x4ed, 0x4ed})
    	/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x14198012bc50 sp=0x14198012bbc0 pc=0x228c838
    github.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x14198012bcf0?, {0x141980034b13?, 0x1000?, 0x0?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x14198012bc80 sp=0x14198012bc50 pc=0x38b33e2
    bufio.(*Reader).fill(0x141980807010)
    	/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14198012bcb8 sp=0x14198012bc80 pc=0x20a6c63
    bufio.(*Reader).ReadSlice(0x141980807010, 0xa)
    	/usr/lib/golang/src/bufio/bufio.go:377 +0x29 fp=0x14198012bd00 sp=0x14198012bcb8 pc=0x20a7769
    bufio.(*Reader).collectFragments(0x141980807010, 0xa)
    	/usr/lib/golang/src/bufio/bufio.go:452 +0x85 fp=0x14198012bde0 sp=0x14198012bd00 pc=0x20a7b85
    bufio.(*Reader).ReadBytes(0x4bbc5a0?, 0xa0?)
    	/usr/lib/golang/src/bufio/bufio.go:479 +0x17 fp=0x14198012be60 sp=0x14198012bde0 pc=0x20a7e77
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.(*frameReader).advance(0x141980807000)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:64 +0x45 fp=0x14198012beb0 sp=0x14198012be60 pc=0x38d6d45
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readRoutine(0x141980807000, 0x14197f8c8370)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:85 +0x51 fp=0x14198012bfc0 sp=0x14198012beb0 pc=0x38d7591
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap2()
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x1b fp=0x14198012bfe0 sp=0x14198012bfc0 pc=0x38d74db
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14198012bfe8 sp=0x14198012bfe0 pc=0x1f40801
    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 184
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x24a
    
    goroutine 183 gp=0x14197fd3d860 m=nil [semacquire]:
    runtime.gopark(0x7f8090f36c08?, 0x1eed7e0?, 0x80?, 0x43?, 0x2?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fac5208 sp=0x14197fac51e8 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fac5270 sp=0x14197fac5208 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x14197fac5310 sp=0x14197fac5270 pc=0x1edbb12
    runtime.gcAssistAlloc(0x14197fd3d860)
    	/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x14197fac53a0 sp=0x14197fac5310 pc=0x1ee06b6
    runtime.deductAssistCredit(0x14197fac53f0?)
    	/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x14197fac53c0 sp=0x14197fac53a0 pc=0x1ed10a6
    runtime.mallocgc(0x5500, 0x0, 0x0)
    	/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x14197fac53f0 sp=0x14197fac53c0 pc=0x1f35e07
    runtime.growslice(0x141983adc000, 0x14197fac54a8?, 0x1ed3be5?, 0x82bed80?, 0x1ed0417?)
    	/usr/lib/golang/src/runtime/slice.go:265 +0x573 fp=0x14197fac5460 sp=0x14197fac53f0 pc=0x1f3ba93
    strings.(*Builder).Write(0x14197f569ea0, {0x1419839fd000, 0x1000, 0x1000000010765?})
    	/usr/lib/golang/src/strings/builder.go:89 +0x90 fp=0x14197fac54b0 sp=0x14197fac5460 pc=0x1ff0cb0
    bufio.(*Writer).Flush(0x14197fac5698)
    	/usr/lib/golang/src/bufio/bufio.go:642 +0x52 fp=0x14197fac54f8 sp=0x14197fac54b0 pc=0x20a8832
    bufio.(*Writer).WriteString(0x14197fac5698, {0x553a4f6?, 0x553a9e6?})
    	/usr/lib/golang/src/bufio/bufio.go:766 +0x165 fp=0x14197fac5568 sp=0x14197fac54f8 pc=0x20a8f65
    encoding/csv.(*Writer).Write(0x14197fac5688, {0x14198384d480?, 0x4, 0x141983909500?})
    	/usr/lib/golang/src/encoding/csv/writer.go:92 +0x2b4 fp=0x14197fac55d0 sp=0x14197fac5568 pc=0x20ad3d4
    github.com/Azure/azure-kusto-go/azkustodata/query.(*row).String(0x5e?)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/row_impl.go:101 +0x21f fp=0x14197fac56e8 sp=0x14197fac55d0 pc=0x38cf67f
    fmt.(*pp).handleMethods(0x14197fc20000, 0x7fac58f0?)
    	/usr/lib/golang/src/fmt/print.go:674 +0x2f4 fp=0x14197fac58a8 sp=0x14197fac56e8 pc=0x1fdd1d4
    fmt.(*pp).printArg(0x14197fc20000, {0x549a3a0, 0x14197fd92e80}, 0x76)
    	/usr/lib/golang/src/fmt/print.go:757 +0x42e fp=0x14197fac5938 sp=0x14197fac58a8 pc=0x1fdd9ee
    fmt.(*pp).doPrintf(0x14197fc20000, {0x553a400, 0x2}, {0x14197fac5d08, 0x1, 0x1})
    	/usr/lib/golang/src/fmt/print.go:1075 +0x3ec fp=0x14197fac5a20 sp=0x14197fac5938 pc=0x1fe04ec
    fmt.Sprintf({0x553a400, 0x2}, {0x14197fac5d08, 0x1, 0x1})
    	/usr/lib/golang/src/fmt/print.go:239 +0x53 fp=0x14197fac5a78 sp=0x14197fac5a20 pc=0x1fda6b3
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197fc4b200}, 0x14197f1c02a0)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:160 +0x8d8 fp=0x14197fac5d60 sp=0x14197fac5a78 pc=0x391bad8
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x14197fdd8140?}, {0x5a00300?, 0x14197fc4b200?}, 0x0?)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x14197fac5e88 sp=0x14197fac5d60 pc=0x3929c16
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14197fac5f78 sp=0x14197fac5e88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197fac5fe0 sp=0x14197fac5f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fac5fe8 sp=0x14197fac5fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 184 gp=0x14197fd3da40 m=nil [chan receive]:
    runtime.gopark(0x14197f3c29a0?, 0x2034990?, 0x10?, 0xea?, 0x59f0ad0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14198011f9d8 sp=0x14198011f9b8 pc=0x1f38d2e
    runtime.chanrecv(0x14197f3fa700, 0x14198011fcb8, 0x1)
    	/usr/lib/golang/src/runtime/chan.go:667 +0x4ae fp=0x14198011fa50 sp=0x14198011f9d8 pc=0x1ec7d2e
    runtime.chanrecv2(0x5a09ed0?, 0x14197f530f90?)
    	/usr/lib/golang/src/runtime/chan.go:514 +0x12 fp=0x14198011fa78 sp=0x14198011fa50 pc=0x1ec7872
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197fc4b340}, 0x14197f1c02a0)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:141 +0x94d fp=0x14198011fd60 sp=0x14198011fa78 pc=0x391bb4d
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x74666968736e6570?, {0x59f0b40?, 0x14197fdd8140?}, {0x5a00300?, 0x14197fc4b340?}, 0x6d736f63222c2273?)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x14198011fe88 sp=0x14198011fd60 pc=0x3929c16
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14198011ff78 sp=0x14198011fe88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14198011ffe0 sp=0x14198011ff78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14198011ffe8 sp=0x14198011ffe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 135 gp=0x14197fb465a0 m=nil [semacquire]:
    runtime.gopark(0x14197f454000?, 0x22dc6c0?, 0xa0?, 0x82?, 0x1419808322a0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fe99ce8 sp=0x14197fe99cc8 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fe99d50 sp=0x14197fe99ce8 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x14197fe99df0 sp=0x14197fe99d50 pc=0x1edbb12
    runtime.gcAssistAlloc(0x14197fb465a0)
    	/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x14197fe99e80 sp=0x14197fe99df0 pc=0x1ee06b6
    runtime.deductAssistCredit(0x0?)
    	/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x14197fe99ea0 sp=0x14197fe99e80 pc=0x1ed10a6
    runtime.mallocgc(0x70, 0x0, 0x1)
    	/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x14197fe99ed0 sp=0x14197fe99ea0 pc=0x1f35e07
    runtime.makechan(0x0?, 0x0)
    	/usr/lib/golang/src/runtime/chan.go:99 +0x126 fp=0x14197fe99f10 sp=0x14197fe99ed0 pc=0x1ec6826
    net/http.(*http2clientStream).cleanupWriteRequest(0x14197f2f1200, {0x0, 0x0})
    	/usr/lib/golang/src/net/http/h2_bundle.go:8951 +0xd6 fp=0x14197fe99f90 sp=0x14197fe99f10 pc=0x22db1b6
    net/http.(*http2clientStream).doRequest(0x14197f2f1200, 0x14197f7480a0?, 0x14197f4d0020?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8719 +0x28 fp=0x14197fe99fb8 sp=0x14197fe99f90 pc=0x22d99e8
    net/http.(*http2ClientConn).roundTrip.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197fe99fe0 sp=0x14197fe99fb8 pc=0x22d999f
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fe99fe8 sp=0x14197fe99fe0 pc=0x1f40801
    created by net/http.(*http2ClientConn).roundTrip in goroutine 110
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470
    
    goroutine 90 gp=0x14197fd82b40 m=nil [select]:
    runtime.gopark(0x14197ff1df08?, 0x6?, 0x40?, 0x2b?, 0x14197ff1dda4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f203c08 sp=0x14197f203be8 pc=0x1f38d2e
    runtime.selectgo(0x14197f203f08, 0x14197ff1dd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f203d38 sp=0x14197f203c08 pc=0x1f152a5
    net/http.(*http2clientStream).writeRequest(0x14197f2f0f00, 0x14197fb50140, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x14197f203f90 sp=0x14197f203d38 pc=0x22da6a8
    net/http.(*http2clientStream).doRequest(0x14197f2f0f00, 0x0?, 0x14197f1f1f98?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x14197f203fb8 sp=0x14197f203f90 pc=0x22d99d8
    net/http.(*http2ClientConn).roundTrip.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197f203fe0 sp=0x14197f203fb8 pc=0x22d999f
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f203fe8 sp=0x14197f203fe0 pc=0x1f40801
    created by net/http.(*http2ClientConn).roundTrip in goroutine 183
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470
    
    goroutine 225 gp=0x14197fd834a0 m=nil [sync.Cond.Wait]:
    runtime.gopark(0x22b8a73?, 0x141980390000?, 0x0?, 0x60?, 0xe?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419801298f8 sp=0x1419801298d8 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    sync.runtime_notifyListWait(0x14197f2f0f48, 0x3c)
    	/usr/lib/golang/src/runtime/sema.go:617 +0x1b3 fp=0x141980129948 sp=0x1419801298f8 pc=0x1f3a8f3
    sync.(*Cond).Wait(0x14197fbb1640?)
    	/usr/lib/golang/src/sync/cond.go:71 +0x73 fp=0x141980129980 sp=0x141980129948 pc=0x1f4d673
    net/http.(*http2pipe).Read(0x14197f2f0f30, {0x14197fd98000, 0x1000, 0x1000})
    	/usr/lib/golang/src/net/http/h2_bundle.go:3928 +0xdb fp=0x1419801299e8 sp=0x141980129980 pc=0x22c2c1b
    net/http.http2transportResponseBody.Read({0x141980129a60?}, {0x14197fd98000?, 0x20a6c63?, 0x1f3ec12?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:9852 +0x59 fp=0x141980129a70 sp=0x1419801299e8 pc=0x22dfdd9
    bufio.(*Reader).fill(0x14197fc90180)
    	/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x141980129aa8 sp=0x141980129a70 pc=0x20a6c63
    bufio.(*Reader).ReadByte(0x14197fc90180)
    	/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x141980129ac0 sp=0x141980129aa8 pc=0x20a73c7
    compress/flate.(*decompressor).huffSym(0x141980058608, 0x141980058638)
    	/usr/lib/golang/src/compress/flate/inflate.go:720 +0x7e fp=0x141980129b10 sp=0x141980129ac0 pc=0x228893e
    compress/flate.(*decompressor).huffmanBlock(0x82bed80?)
    	/usr/lib/golang/src/compress/flate/inflate.go:495 +0x45 fp=0x141980129b90 sp=0x141980129b10 pc=0x2287605
    compress/flate.(*decompressor).Read(0x141980058608, {0x14197fd99cea, 0x316, 0xcea?})
    	/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x141980129bc0 sp=0x141980129b90 pc=0x2286dbb
    compress/gzip.(*Reader).Read(0x14197f3d8848, {0x14197fd99cea, 0x316, 0x316})
    	/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x141980129c50 sp=0x141980129bc0 pc=0x228c838
    github.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x141980129cf0?, {0x14197fd99cea?, 0x1000?, 0x0?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x141980129c80 sp=0x141980129c50 pc=0x38b33e2
    bufio.(*Reader).fill(0x14197f546110)
    	/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x141980129cb8 sp=0x141980129c80 pc=0x20a6c63
    bufio.(*Reader).ReadSlice(0x14197f546110, 0xa)
    	/usr/lib/golang/src/bufio/bufio.go:377 +0x29 fp=0x141980129d00 sp=0x141980129cb8 pc=0x20a7769
    bufio.(*Reader).collectFragments(0x14197f546110, 0xa)
    	/usr/lib/golang/src/bufio/bufio.go:452 +0x85 fp=0x141980129de0 sp=0x141980129d00 pc=0x20a7b85
    bufio.(*Reader).ReadBytes(0x4bbc5a0?, 0xa0?)
    	/usr/lib/golang/src/bufio/bufio.go:479 +0x17 fp=0x141980129e60 sp=0x141980129de0 pc=0x20a7e77
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.(*frameReader).advance(0x14197f546100)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:64 +0x45 fp=0x141980129eb0 sp=0x141980129e60 pc=0x38d6d45
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readRoutine(0x14197f546100, 0x14197f8c54f0)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:85 +0x51 fp=0x141980129fc0 sp=0x141980129eb0 pc=0x38d7591
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap2()
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x1b fp=0x141980129fe0 sp=0x141980129fc0 pc=0x38d74db
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x141980129fe8 sp=0x141980129fe0 pc=0x1f40801
    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 183
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x24a
    
    goroutine 174 gp=0x14197ff64000 m=nil [select]:
    runtime.gopark(0x1419800b9df8?, 0x2?, 0x0?, 0x0?, 0x1419800b9db4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf3c50 sp=0x14197fcf3c30 pc=0x1f38d2e
    runtime.selectgo(0x14197fcf3df8, 0x1419800b9db0, 0x59d87c8?, 0x0, 0x14197f2980e0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197fcf3d80 sp=0x14197fcf3c50 pc=0x1f152a5
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.nextFrame(0x14197f8c8370)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:198 +0x96 fp=0x14197fcf3e38 sp=0x14197fcf3d80 pc=0x38d7d16
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readPrimaryTable(0x14197f8c8370, 0x14197fb50000)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:258 +0xb1 fp=0x14197fcf3ec8 sp=0x14197fcf3e38 pc=0x38d8171
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x14197f8c8370)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:174 +0x189 fp=0x14197fcf3f30 sp=0x14197fcf3ec8 pc=0x38d7b89
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x14197f8c8370, 0x14197f59aee0)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x14197fcf3fc0 sp=0x14197fcf3f30 pc=0x38d788c
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x14197fcf3fe0 sp=0x14197fcf3fc0 pc=0x38d751b
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf3fe8 sp=0x14197fcf3fe0 pc=0x1f40801
    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 184
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc
    
    rax    0x0
    rbx    0x7f808bfff640
    rcx    0x7f80df60594c
    rdx    0x6
    rdi    0x109fd
    rsi    0x10a00
    rbp    0x10a00
    rsp    0x7f808bffe790
    r8     0x7f808bffe860
    r9     0x7f80df72b4e0
    r10    0x8
    r11    0x246
    r12    0x6
    r13    0x7f808bffeb60
    r14    0x14197f1334a0
    r15    0xffffffffffffffff
    rip    0x7f80df60594c
    rflags 0x246
    cs     0x33
    fs     0x0
    gs     0x0
    
      query-infra-svc: command failed: exit status 2
    output: runtime/cgo: pthread_create failed: Resource temporarily unavailable
    SIGABRT: abort
    PC=0x7f7ef9baa94c m=3 sigcode=18446744073709551610
    
    goroutine 0 gp=0x170ac5c1f0e0 m=3 mp=0x170ac5c91808 [idle]:
    runtime: g 0 gp=0x170ac5c1f0e0: unknown pc 0x7f7ef9baa94c
    stack: frame={sp:0x7f7eabffe790, fp:0x0} stack=[0x7f7eab7ff1e0,0x7f7eabffede0)
                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef
    00007f7eabffe690: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6a0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6b0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6c0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6d0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6e0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6f0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe700: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe710: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe720: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe730: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe740: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe750: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe760: 00000000 00000000  00007f7e f9bb7fa9  ............~...
    00007f7eabffe770: 00000000 00000000  00000000 00000011  ................
    00007f7eabffe780: 00000000 00000130  00007f7e f9baa93e  0.......>...~...
    00007f7eabffe790: 00000000 00000120  00000000 00000000   ...............
                      ^ SP
    00007f7eabffe7a0: 00000000 00000013  00000000 00000000  ................
    00007f7eabffe7b0: 00000000 00000000  00007f7e f9c2dcff  ............~...
    00007f7eabffe7c0: 00007f7e 7bfff910  00007f7e f9c2dcb1  ...{~.......~...
    00007f7eabffe7d0: 00007f7e 7bfff640  00007f7e abffeaa0  @..{~.......~...
    00007f7eabffe7e0: 00007f7e abffe90e  00007f7e abffe90f  ....~.......~...
    00007f7eabffe7f0: 00007f7e 7bfff640  00007f7e f9ba87f5  @..{~.......~...
    00007f7eabffe800: 00007f7e a4000020  00000000 00000120   ...~... .......
    00007f7eabffe810: 00000000 003d0f00  176a34c4 bac87500  ..=......u...4j.
    00007f7eabffe820: 00007f7e abfff640  00000000 00000006  @...~...........
    00007f7eabffe830: 00007f7e a4000f30  00007f7e abffeb60  0...~...`...~...
    00007f7eabffe840: 0000170a c5c1f0e0  00007f7e f9b5d646  ........F...~...
    00007f7eabffe850: 00007f7e f9d1ae90  00007f7e f9b477f3  ....~....w..~...
    00007f7eabffe860: 00000000 00000020  176a34c4 bac87500   ........u...4j.
    00007f7eabffe870: 00007f7e 7b7ff000  00007f7e f9c1cac4  ...{~.......~...
    00007f7eabffe880: 00000000 00000000  00000000 0000000d  ................
    runtime: g 0 gp=0x170ac5c1f0e0: unknown pc 0x7f7ef9baa94c
    stack: frame={sp:0x7f7eabffe790, fp:0x0} stack=[0x7f7eab7ff1e0,0x7f7eabffede0)
                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef
    00007f7eabffe690: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6a0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6b0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6c0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6d0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6e0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6f0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe700: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe710: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe720: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe730: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe740: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe750: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe760: 00000000 00000000  00007f7e f9bb7fa9  ............~...
    00007f7eabffe770: 00000000 00000000  00000000 00000011  ................
    00007f7eabffe780: 00000000 00000130  00007f7e f9baa93e  0.......>...~...
    00007f7eabffe790: 00000000 00000120  00000000 00000000   ...............
                      ^ SP
    00007f7eabffe7a0: 00000000 00000013  00000000 00000000  ................
    00007f7eabffe7b0: 00000000 00000000  00007f7e f9c2dcff  ............~...
    00007f7eabffe7c0: 00007f7e 7bfff910  00007f7e f9c2dcb1  ...{~.......~...
    00007f7eabffe7d0: 00007f7e 7bfff640  00007f7e abffeaa0  @..{~.......~...
    00007f7eabffe7e0: 00007f7e abffe90e  00007f7e abffe90f  ....~.......~...
    00007f7eabffe7f0: 00007f7e 7bfff640  00007f7e f9ba87f5  @..{~.......~...
    00007f7eabffe800: 00007f7e a4000020  00000000 00000120   ...~... .......
    00007f7eabffe810: 00000000 003d0f00  176a34c4 bac87500  ..=......u...4j.
    00007f7eabffe820: 00007f7e abfff640  00000000 00000006  @...~...........
    00007f7eabffe830: 00007f7e a4000f30  00007f7e abffeb60  0...~...`...~...
    00007f7eabffe840: 0000170a c5c1f0e0  00007f7e f9b5d646  ........F...~...
    00007f7eabffe850: 00007f7e f9d1ae90  00007f7e f9b477f3  ....~....w..~...
    00007f7eabffe860: 00000000 00000020  176a34c4 bac87500   ........u...4j.
    00007f7eabffe870: 00007f7e 7b7ff000  00007f7e f9c1cac4  ...{~.......~...
    00007f7eabffe880: 00000000 00000000  00000000 0000000d  ................
    
    goroutine 1 gp=0x170ac5c1e1e0 m=nil [sync.WaitGroup.Wait]:
    runtime.gopark(0x170ac5cb27b8?, 0x2022a4d?, 0x70?, 0x8e?, 0x170ac5cb27e8?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cb27a0 sp=0x170ac5cb2780 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x170ac65f8210, 0x0, 0x1, 0x0, 0x19)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x170ac5cb2808 sp=0x170ac5cb27a0 pc=0x1f162b2
    sync.runtime_SemacquireWaitGroup(0x170ac6709950?, 0xe0?)
    	/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x170ac5cb2840 sp=0x170ac5cb2808 pc=0x1f3a66e
    sync.(*WaitGroup).Wait(0x170ac65f8208)
    	/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x170ac5cb2868 sp=0x170ac5cb2840 pc=0x1f4f445
    golang.org/x/sync/errgroup.(*Group).Wait(0x170ac65f8200)
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x170ac5cb2888 sp=0x170ac5cb2868 pc=0x362507e
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0}, {0x170ac626a1e0, 0x6, 0x6})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x170ac5cb2948 sp=0x170ac5cb2888 pc=0x3927e05
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).gatherInfraLogs(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:595 +0xadd fp=0x170ac5cb2d30 sp=0x170ac5cb2948 pc=0x392799d
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:333 +0x533 fp=0x170ac5cb3908 sp=0x170ac5cb2d30 pc=0x3921bf3
    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedInfraQueryOptions).RunInfra(0x170ac5c2c108, {0x59f0b08, 0x170ac64108d0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:83 +0x71f fp=0x170ac5cb3b30 sp=0x170ac5cb3908 pc=0x393887f
    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawInfraQueryOptions).Run(0x0?, {0x59f0b08, 0x170ac64108d0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_options.go:109 +0x54 fp=0x170ac5cb3b58 sp=0x170ac5cb3b30 pc=0x3939134
    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryInfraCommand.func1(0x170ac66f5508?, {0x170ac5d40960?, 0x4?, 0x553b0e2?})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:45 +0x28 fp=0x170ac5cb3b80 sp=0x170ac5cb3b58 pc=0x3938128
    github.com/spf13/cobra.(*Command).execute(0x170ac66f5508, {0x170ac5e36100, 0xa, 0xa})
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x170ac5cb3d28 sp=0x170ac5cb3b80 pc=0x212e614
    github.com/spf13/cobra.(*Command).ExecuteC(0x170ac6228608)
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x170ac5cb3e18 sp=0x170ac5cb3d28 pc=0x212efa5
    github.com/spf13/cobra.(*Command).Execute(...)
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071
    github.com/spf13/cobra.(*Command).ExecuteContext(...)
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064
    main.main()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x170ac5cb3f48 sp=0x170ac5cb3e18 pc=0x47ddd6a
    runtime.main()
    	/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x170ac5cb3fe0 sp=0x170ac5cb3f48 pc=0x1f01975
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cb3fe8 sp=0x170ac5cb3fe0 pc=0x1f40801
    
    goroutine 2 gp=0x170ac5c1ed20 m=nil [force gc (idle)]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdefa8 sp=0x170ac5cdef88 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.forcegchelper()
    	/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x170ac5cdefe0 sp=0x170ac5cdefa8 pc=0x1f01c93
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdefe8 sp=0x170ac5cdefe0 pc=0x1f40801
    created by runtime.init.7 in goroutine 1
    	/usr/lib/golang/src/runtime/proc.go:363 +0x1a
    
    goroutine 18 gp=0x170ac5d843c0 m=nil [GC sweep wait]:
    runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cda788 sp=0x170ac5cda768 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.bgsweep(0x170ac5d92000)
    	/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x170ac5cda7c8 sp=0x170ac5cda788 pc=0x1ee9f11
    runtime.gcenable.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x170ac5cda7e0 sp=0x170ac5cda7c8 pc=0x1edadd7
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cda7e8 sp=0x170ac5cda7e0 pc=0x1f40801
    created by runtime.gcenable in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:214 +0x66
    
    goroutine 19 gp=0x170ac5d845a0 m=nil [GC scavenge wait]:
    runtime.gopark(0x10000?, 0x5981cd8?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdaf78 sp=0x170ac5cdaf58 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.(*scavengerState).park(0x82979c0)
    	/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x170ac5cdafa8 sp=0x170ac5cdaf78 pc=0x1ee77e9
    runtime.bgscavenge(0x170ac5d92000)
    	/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x170ac5cdafc8 sp=0x170ac5cdafa8 pc=0x1ee7d99
    runtime.gcenable.gowrap2()
    	/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x170ac5cdafe0 sp=0x170ac5cdafc8 pc=0x1edad97
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdafe8 sp=0x170ac5cdafe0 pc=0x1f40801
    created by runtime.gcenable in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:215 +0xa5
    
    goroutine 34 gp=0x170ac5e00000 m=nil [GOMAXPROCS updater (idle)]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e06788 sp=0x170ac5e06768 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.updateMaxProcsGoroutine()
    	/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x170ac5e067e0 sp=0x170ac5e06788 pc=0x1f10267
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e067e8 sp=0x170ac5e067e0 pc=0x1f40801
    created by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1
    	/usr/lib/golang/src/runtime/proc.go:7083 +0x37
    
    goroutine 35 gp=0x170ac5e005a0 m=nil [finalizer wait]:
    runtime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x170ac5cde601?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cde620 sp=0x170ac5cde600 pc=0x1f38d2e
    runtime.runFinalizers()
    	/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x170ac5cde7e0 sp=0x170ac5cde620 pc=0x1ed9cc7
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cde7e8 sp=0x170ac5cde7e0 pc=0x1f40801
    created by runtime.createfing in goroutine 1
    	/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d
    
    goroutine 36 gp=0x170ac5ff81e0 m=nil [cleanup wait]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e06f68 sp=0x170ac5e06f48 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.(*cleanupQueue).dequeue(0x82980e0)
    	/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x170ac5e06fa0 sp=0x170ac5e06f68 pc=0x1ed6074
    runtime.runCleanups()
    	/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x170ac5e06fe0 sp=0x170ac5e06fa0 pc=0x1ed66e5
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e06fe8 sp=0x170ac5e06fe0 pc=0x1f40801
    created by runtime.(*cleanupQueue).createGs in goroutine 1
    	/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5
    
    goroutine 37 gp=0x170ac611bc20 m=nil [GC worker (idle)]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e07740 sp=0x170ac5e07720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x170ac5e0b9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5e077c8 sp=0x170ac5e07740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5e077e0 sp=0x170ac5e077c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e077e8 sp=0x170ac5e077e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 3 gp=0x170ac5c1f680 m=nil [runnable]:
    runtime.gopark(0x125c26559f5b?, 0x3?, 0x94?, 0x85?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdf740 sp=0x170ac5cdf720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x170ac5e0b9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdf7c8 sp=0x170ac5cdf740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdf7e0 sp=0x170ac5cdf7c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdf7e8 sp=0x170ac5cdf7e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 20 gp=0x170ac5d84780 m=nil [GC worker (idle)]:
    runtime.gopark(0x125c265c3db1?, 0x3?, 0x92?, 0xcd?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdb740 sp=0x170ac5cdb720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x170ac5e0b9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdb7c8 sp=0x170ac5cdb740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdb7e0 sp=0x170ac5cdb7c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdb7e8 sp=0x170ac5cdb7e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 38 gp=0x170ac6134000 m=nil [GC worker (idle)]:
    runtime.gopark(0x125c26588b9e?, 0x1?, 0xf7?, 0xb4?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e07f40 sp=0x170ac5e07f20 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x170ac5e0b9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5e07fc8 sp=0x170ac5e07f40 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5e07fe0 sp=0x170ac5e07fc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e07fe8 sp=0x170ac5e07fe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 4 gp=0x170ac5c1f860 m=10 mp=0x170ac6280008 [GC mark termination]:
    runtime.systemstack_switch()
    	/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x170ac5cdfea0 sp=0x170ac5cdfe90 pc=0x1f3ebc8
    runtime.forEachP(...)
    	/usr/lib/golang/src/runtime/proc.go:2112
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x170ac5cdff40 sp=0x170ac5cdfea0 pc=0x1edbc34
    runtime.gcBgMarkWorker(0x170ac5e0b9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x170ac5cdffc8 sp=0x170ac5cdff40 pc=0x1edda88
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdffe0 sp=0x170ac5cdffc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdffe8 sp=0x170ac5cdffe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 21 gp=0x170ac5d84960 m=nil [GC worker (idle)]:
    runtime.gopark(0x125c265818cf?, 0x3?, 0xa5?, 0x9c?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdbf40 sp=0x170ac5cdbf20 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x170ac5e0b9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdbfc8 sp=0x170ac5cdbf40 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdbfe0 sp=0x170ac5cdbfc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdbfe8 sp=0x170ac5cdbfe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 22 gp=0x170ac5d84b40 m=nil [GC worker (idle)]:
    runtime.gopark(0x125c265846e7?, 0x3?, 0x77?, 0xb6?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdc740 sp=0x170ac5cdc720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x170ac5e0b9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdc7c8 sp=0x170ac5cdc740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdc7e0 sp=0x170ac5cdc7c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdc7e8 sp=0x170ac5cdc7e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 23 gp=0x170ac5d84d20 m=nil [GC worker (idle)]:
    runtime.gopark(0x125c26581951?, 0x3?, 0x91?, 0x58?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdcf40 sp=0x170ac5cdcf20 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x170ac5e0b9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdcfc8 sp=0x170ac5cdcf40 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdcfe0 sp=0x170ac5cdcfc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdcfe8 sp=0x170ac5cdcfe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 39 gp=0x170ac648a3c0 m=nil [select, locked to thread]:
    runtime.gopark(0x170ac5e097a8?, 0x2?, 0x1?, 0xa3?, 0x170ac5e09794?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e09628 sp=0x170ac5e09608 pc=0x1f38d2e
    runtime.selectgo(0x170ac5e097a8, 0x170ac5e09790, 0x0?, 0x0, 0x0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5e09758 sp=0x170ac5e09628 pc=0x1f152a5
    runtime.ensureSigM.func1()
    	/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x170ac5e097e0 sp=0x170ac5e09758 pc=0x1f328e8
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e097e8 sp=0x170ac5e097e0 pc=0x1f40801
    created by runtime.ensureSigM in goroutine 1
    	/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5
    
    goroutine 24 gp=0x170ac62f4960 m=8 mp=0x170ac6180008 [syscall]:
    runtime.notetsleepg(0x82bdea0, 0xffffffffffffffff)
    	/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x170ac5e037a0 sp=0x170ac5e03778 pc=0x1ece509
    os/signal.signal_recv()
    	/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x170ac5e037c0 sp=0x170ac5e037a0 pc=0x1f3b298
    os/signal.loop()
    	/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x170ac5e037e0 sp=0x170ac5e037c0 pc=0x2030553
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e037e8 sp=0x170ac5e037e0 pc=0x1f40801
    created by os/signal.Notify.func1.1 in goroutine 1
    	/usr/lib/golang/src/os/signal/signal.go:152 +0x1f
    
    goroutine 50 gp=0x170ac6135a40 m=nil [select]:
    runtime.gopark(0x170ac66e3798?, 0x2?, 0x80?, 0x6f?, 0x170ac66e378c?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac66e3628 sp=0x170ac66e3608 pc=0x1f38d2e
    runtime.selectgo(0x170ac66e3798, 0x170ac66e3788, 0x0?, 0x0, 0x0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac66e3758 sp=0x170ac66e3628 pc=0x1f152a5
    os/signal.NotifyContext.func1()
    	/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x170ac66e37e0 sp=0x170ac66e3758 pc=0x202ff9c
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac66e37e8 sp=0x170ac66e37e0 pc=0x1f40801
    created by os/signal.NotifyContext in goroutine 1
    	/usr/lib/golang/src/os/signal/signal.go:291 +0x158
    
    goroutine 102 gp=0x170ac61843c0 m=nil [select]:
    runtime.gopark(0x170ac5cefc70?, 0x2?, 0xc8?, 0xf9?, 0x170ac5cefb6c?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cef9f0 sp=0x170ac5cef9d0 pc=0x1f38d2e
    runtime.selectgo(0x170ac5cefc70, 0x170ac5cefb68, 0x554857c?, 0x0, 0x226?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5cefb20 sp=0x170ac5cef9f0 pc=0x1f152a5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x170ac5c49130}, 0x170ac65fa070, 0x170ac6410cf0)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:283 +0x21b fp=0x170ac5ceff48 sp=0x170ac5cefb20 pc=0x39204fb
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x170ac5ceff78 sp=0x170ac5ceff48 pc=0x392800c
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5ceffe0 sp=0x170ac5ceff78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5ceffe8 sp=0x170ac5ceffe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 104 gp=0x170ac6184d20 m=nil [select]:
    runtime.gopark(0x170ac6992ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69929d8?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6992848 sp=0x170ac6992828 pc=0x1f38d2e
    runtime.selectgo(0x170ac6992ac8, 0x170ac69929d0, 0x55562e1?, 0x0, 0x1f6d6cf?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6992978 sp=0x170ac6992848 pc=0x1f152a5
    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac619fb80, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac6992b40 sp=0x170ac6992978 pc=0x22d92e5
    net/http.(*http2ClientConn).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8603
    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac619fb80, {0x4c?, 0xfa?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac6992c80 sp=0x170ac6992b40 pc=0x22d50ad
    net/http.(*http2Transport).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:7852
    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac619fb80?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac6992cc0 sp=0x170ac6992c80 pc=0x22e3a9c
    net/http.(*Transport).roundTrip(0x8175400, 0x170ac619fa40)
    	/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac6992ea0 sp=0x170ac6992cc0 pc=0x231037f
    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)
    	/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac6992ec0 sp=0x170ac6992ea0 pc=0x2321178
    net/http.send(0x170ac619fa40, {0x59a1ce0, 0x8175400}, {0x170ac6993138?, 0x1f3bac6?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69930b0 sp=0x170ac6992ec0 pc=0x22ad9ab
    net/http.(*Client).send(0x170ac6410c60, 0x170ac619fa40, {0x170ac69931e8?, 0x1fdd787?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac6993148 sp=0x170ac69930b0 pc=0x22ad1d8
    net/http.(*Client).do(0x170ac6410c60, 0x170ac619fa40)
    	/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac6993338 sp=0x170ac6993148 pc=0x22af817
    net/http.(*Client).Do(...)
    	/usr/lib/golang/src/net/http/client.go:592
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5c49450}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e34a0}, 0x170ac65f7ce0, {0x170ac65d9560, 0x112})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac6993630 sp=0x170ac6993338 pc=0x38de6bd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5c49450}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x170ac65f7c80, 0x0, {0x0, ...}, ...})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac6993888 sp=0x170ac6993630 pc=0x38ddecd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5c49450?}, 0xa8?, {0x170ac5db1a50?, 0x170ac6764a48?}, 0x3?, 0x2022a4d?)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac6993960 sp=0x170ac6993888 pc=0x38dd40b
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194700?}, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x170ac6764a48, 0x3, 0x4})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69939e0 sp=0x170ac6993960 pc=0x38e51e2
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194700}, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x0?, 0x2?, 0x1?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac6993a78 sp=0x170ac69939e0 pc=0x38e4eaf
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020500}, 0x170ac65fa000)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac6993d60 sp=0x170ac6993a78 pc=0x391b5c5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac5fe1880?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020500?}, 0x0?)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac6993e88 sp=0x170ac6993d60 pc=0x3929c16
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac6993f78 sp=0x170ac6993e88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac6993fe0 sp=0x170ac6993f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6993fe8 sp=0x170ac6993fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 101 gp=0x170ac61d4000 m=nil [sync.WaitGroup.Wait]:
    runtime.gopark(0x170ac5de7e00?, 0x1ed0765?, 0xb0?, 0xfa?, 0xe?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de7da0 sp=0x170ac5de7d80 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x170ac65f8290, 0x0, 0x1, 0x0, 0x19)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x170ac5de7e08 sp=0x170ac5de7da0 pc=0x1f162b2
    sync.runtime_SemacquireWaitGroup(0x170ac6709a70?, 0x0?)
    	/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x170ac5de7e40 sp=0x170ac5de7e08 pc=0x1f3a66e
    sync.(*WaitGroup).Wait(0x170ac65f8288)
    	/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x170ac5de7e68 sp=0x170ac5de7e40 pc=0x1f4f445
    golang.org/x/sync/errgroup.(*Group).Wait(0x170ac65f8280)
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x170ac5de7e88 sp=0x170ac5de7e68 pc=0x362507e
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x170ac5fe1140, {0x59f0b40, 0x170ac5c49130}, {0x170ac626a1e0, 0x6, 0x170ac5cac1c0?}, 0x170ac65fa000)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x170ac5de7f00 sp=0x170ac5de7e88 pc=0x392976a
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x170ac5de7f78 sp=0x170ac5de7f00 pc=0x392808e
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5de7fe0 sp=0x170ac5de7f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de7fe8 sp=0x170ac5de7fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 67 gp=0x170ac62f5a40 m=nil [IO wait]:
    runtime.gopark(0x170ac6497780?, 0x8?, 0x40?, 0x16?, 0xb?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de47b8 sp=0x170ac5de4798 pc=0x1f38d2e
    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)
    	/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x170ac5de47f0 sp=0x170ac5de47b8 pc=0x1ef9e57
    internal/poll.runtime_pollWait(0x7f7eb1670000, 0x72)
    	/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x170ac5de4810 sp=0x170ac5de47f0 pc=0x1f375c5
    internal/poll.(*pollDesc).wait(0x170ac620b480?, 0x170ac65a2000?, 0x0)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x170ac5de4838 sp=0x170ac5de4810 pc=0x1fbd167
    internal/poll.(*pollDesc).waitRead(...)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89
    internal/poll.(*FD).Read(0x170ac620b480, {0x170ac65a2000, 0x3500, 0x3500})
    	/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x170ac5de48d0 sp=0x170ac5de4838 pc=0x1fbe4ce
    net.(*netFD).Read(0x170ac620b480, {0x170ac65a2000?, 0x170ac5de4901?, 0x22a2e75?})
    	/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x170ac5de4918 sp=0x170ac5de48d0 pc=0x20798c5
    net.(*conn).Read(0x170ac62823d8, {0x170ac65a2000?, 0x7f7eb047fc00?, 0x7f7ef9af2e00?})
    	/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x170ac5de4960 sp=0x170ac5de4918 pc=0x208b7a5
    crypto/tls.(*atLeastReader).Read(0x170ac6708f30, {0x170ac65a2000?, 0x170ac5de4a48?, 0x1efcc28?})
    	/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x170ac5de49a8 sp=0x170ac5de4960 pc=0x2222ddb
    bytes.(*Buffer).ReadFrom(0x170ac605fb28, {0x59a3680, 0x170ac6708f30})
    	/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x170ac5de4a00 sp=0x170ac5de49a8 pc=0x1fe9258
    crypto/tls.(*Conn).readFromUntil(0x170ac605f888, {0x59a32c0, 0x170ac62823d8}, 0x222764c?)
    	/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x170ac5de4a38 sp=0x170ac5de4a00 pc=0x2222fbe
    crypto/tls.(*Conn).readRecordOrCCS(0x170ac605f888, 0x0)
    	/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x170ac5de4c18 sp=0x170ac5de4a38 pc=0x222009b
    crypto/tls.(*Conn).readRecord(...)
    	/usr/lib/golang/src/crypto/tls/conn.go:588
    crypto/tls.(*Conn).Read(0x170ac605f888, {0x170ac6029000, 0x1000, 0x22c34e0?})
    	/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x170ac5de4c80 sp=0x170ac5de4c18 pc=0x22267c5
    bufio.(*Reader).Read(0x170ac647aa20, {0x170ac6153624, 0x9, 0x22e100e?})
    	/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0x170ac5de4cb8 sp=0x170ac5de4c80 pc=0x20a71f7
    io.ReadAtLeast({0x59a1260, 0x170ac647aa20}, {0x170ac6153624, 0x9, 0x9}, 0x9)
    	/usr/lib/golang/src/io/io.go:335 +0x8e fp=0x170ac5de4d00 sp=0x170ac5de4cb8 pc=0x1fb17ee
    io.ReadFull(...)
    	/usr/lib/golang/src/io/io.go:354
    net/http.http2readFrameHeader({0x170ac6153624, 0x9, 0x170a00000005?}, {0x59a1260?, 0x170ac647aa20?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:1805 +0x65 fp=0x170ac5de4d50 sp=0x170ac5de4d00 pc=0x22b9e25
    net/http.(*http2Framer).ReadFrameHeader(0x170ac61535e0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:2071 +0x6b fp=0x170ac5de4dd8 sp=0x170ac5de4d50 pc=0x22ba9eb
    net/http.(*http2Framer).ReadFrame(0x170ac61535e0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:2130 +0x18 fp=0x170ac5de4e18 sp=0x170ac5de4dd8 pc=0x22bb1b8
    net/http.(*http2clientConnReadLoop).run(0x170ac5de4fa8)
    	/usr/lib/golang/src/net/http/h2_bundle.go:9550 +0xca fp=0x170ac5de4f60 sp=0x170ac5de4e18 pc=0x22de14a
    net/http.(*http2ClientConn).readLoop(0x170ac61d9880)
    	/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x170ac5de4fc8 sp=0x170ac5de4f60 pc=0x22dd532
    net/http.(*http2Transport).newClientConn.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x170ac5de4fe0 sp=0x170ac5de4fc8 pc=0x22d7057
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de4fe8 sp=0x170ac5de4fe0 pc=0x1f40801
    created by net/http.(*http2Transport).newClientConn in goroutine 66
    	/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5
    
    goroutine 73 gp=0x170ac648ad20 m=nil [IO wait]:
    runtime.gopark(0x170ac6497e40?, 0x1edb176?, 0x80?, 0xed?, 0xb?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de6808 sp=0x170ac5de67e8 pc=0x1f38d2e
    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)
    	/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x170ac5de6840 sp=0x170ac5de6808 pc=0x1ef9e57
    internal/poll.runtime_pollWait(0x7f7eb166fe00, 0x72)
    	/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x170ac5de6860 sp=0x170ac5de6840 pc=0x1f375c5
    internal/poll.(*pollDesc).wait(0x170ac60c4e80?, 0x170ac5ecd300?, 0x0)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x170ac5de6888 sp=0x170ac5de6860 pc=0x1fbd167
    internal/poll.(*pollDesc).waitRead(...)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89
    internal/poll.(*FD).Read(0x170ac60c4e80, {0x170ac5ecd300, 0x1300, 0x1300})
    	/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x170ac5de6920 sp=0x170ac5de6888 pc=0x1fbe4ce
    net.(*netFD).Read(0x170ac60c4e80, {0x170ac5ecd300?, 0x170ac5ecd30a?, 0x5?})
    	/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x170ac5de6968 sp=0x170ac5de6920 pc=0x20798c5
    net.(*conn).Read(0x170ac5d82278, {0x170ac5ecd300?, 0x7f7eb047fc00?, 0x7f7ef9af2e00?})
    	/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x170ac5de69b0 sp=0x170ac5de6968 pc=0x208b7a5
    crypto/tls.(*atLeastReader).Read(0x170ac67088e8, {0x170ac5ecd300?, 0x170ac5de6a98?, 0x1efcc28?})
    	/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x170ac5de69f8 sp=0x170ac5de69b0 pc=0x2222ddb
    bytes.(*Buffer).ReadFrom(0x170ac5dd02a8, {0x59a3680, 0x170ac67088e8})
    	/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x170ac5de6a50 sp=0x170ac5de69f8 pc=0x1fe9258
    crypto/tls.(*Conn).readFromUntil(0x170ac5dd0008, {0x59a32c0, 0x170ac5d82278}, 0x222764c?)
    	/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x170ac5de6a88 sp=0x170ac5de6a50 pc=0x2222fbe
    crypto/tls.(*Conn).readRecordOrCCS(0x170ac5dd0008, 0x0)
    	/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x170ac5de6c68 sp=0x170ac5de6a88 pc=0x222009b
    crypto/tls.(*Conn).readRecord(...)
    	/usr/lib/golang/src/crypto/tls/conn.go:588
    crypto/tls.(*Conn).Read(0x170ac5dd0008, {0x170ac6564000, 0x1000, 0x4e55540?})
    	/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x170ac5de6cd0 sp=0x170ac5de6c68 pc=0x22267c5
    net/http.(*persistConn).Read(0x170ac5f73180, {0x170ac6564000?, 0x59a0ea0?, 0x81260e0?})
    	/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x170ac5de6d30 sp=0x170ac5de6cd0 pc=0x2319d07
    bufio.(*Reader).fill(0x170ac65537a0)
    	/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x170ac5de6d68 sp=0x170ac5de6d30 pc=0x20a6c63
    bufio.(*Reader).Peek(0x170ac65537a0, 0x1)
    	/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x170ac5de6d80 sp=0x170ac5de6d68 pc=0x20a6df3
    net/http.(*persistConn).readLoop(0x170ac5f73180)
    	/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x170ac5de6fc8 sp=0x170ac5de6d80 pc=0x231a812
    net/http.(*Transport).dialConn.gowrap2()
    	/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x170ac5de6fe0 sp=0x170ac5de6fc8 pc=0x2319197
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de6fe8 sp=0x170ac5de6fe0 pc=0x1f40801
    created by net/http.(*Transport).dialConn in goroutine 82
    	/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac
    
    goroutine 103 gp=0x170ac648af00 m=nil [select]:
    runtime.gopark(0x170ac5de0d00?, 0x2?, 0xa0?, 0x6a?, 0x170ac5de0c74?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de0ac8 sp=0x170ac5de0aa8 pc=0x1f38d2e
    runtime.selectgo(0x170ac5de0d00, 0x170ac5de0c70, 0x553bfdc?, 0x0, 0x5a48c80?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5de0bf8 sp=0x170ac5de0ac8 pc=0x1f152a5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x0?, {0x59f0b40, 0x170ac5c49130}, 0x170ac65fa000, 0x170ac65fa070)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x170ac5de0f10 sp=0x170ac5de0bf8 pc=0x392836e
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x170ac5de0f78 sp=0x170ac5de0f10 pc=0x3927f3d
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5de0fe0 sp=0x170ac5de0f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de0fe8 sp=0x170ac5de0fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 74 gp=0x170ac648b4a0 m=nil [select]:
    runtime.gopark(0x170ac5df1f48?, 0x2?, 0x0?, 0xb6?, 0x170ac5df1ef4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6760d88 sp=0x170ac6760d68 pc=0x1f38d2e
    runtime.selectgo(0x170ac6760f48, 0x170ac5df1ef0, 0x170ac68d6980?, 0x0, 0x170ac655e8d0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6760eb8 sp=0x170ac6760d88 pc=0x1f152a5
    net/http.(*persistConn).writeLoop(0x170ac5f73180)
    	/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x170ac6760fc8 sp=0x170ac6760eb8 pc=0x231c426
    net/http.(*Transport).dialConn.gowrap3()
    	/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x170ac6760fe0 sp=0x170ac6760fc8 pc=0x2319157
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6760fe8 sp=0x170ac6760fe0 pc=0x1f40801
    created by net/http.(*Transport).dialConn in goroutine 82
    	/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe
    
    goroutine 105 gp=0x170ac61d43c0 m=nil [select]:
    runtime.gopark(0x170ac699cac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac699c9d8?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac699c848 sp=0x170ac699c828 pc=0x1f38d2e
    runtime.selectgo(0x170ac699cac8, 0x170ac699c9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac699c978 sp=0x170ac699c848 pc=0x1f152a5
    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e000, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac699cb40 sp=0x170ac699c978 pc=0x22d92e5
    net/http.(*http2ClientConn).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8603
    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e000, {0x4c?, 0xfa?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac699cc80 sp=0x170ac699cb40 pc=0x22d50ad
    net/http.(*http2Transport).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:7852
    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e000?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac699ccc0 sp=0x170ac699cc80 pc=0x22e3a9c
    net/http.(*Transport).roundTrip(0x8175400, 0x170ac619fcc0)
    	/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac699cea0 sp=0x170ac699ccc0 pc=0x231037f
    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)
    	/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac699cec0 sp=0x170ac699cea0 pc=0x2321178
    net/http.send(0x170ac619fcc0, {0x59a1ce0, 0x8175400}, {0x170ac699d138?, 0x1f3bac6?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac699d0b0 sp=0x170ac699cec0 pc=0x22ad9ab
    net/http.(*Client).send(0x170ac6410c60, 0x170ac619fcc0, {0x170ac699d1e8?, 0x1fdd787?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac699d148 sp=0x170ac699d0b0 pc=0x22ad1d8
    net/http.(*Client).do(0x170ac6410c60, 0x170ac619fcc0)
    	/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac699d338 sp=0x170ac699d148 pc=0x22af817
    net/http.(*Client).Do(...)
    	/usr/lib/golang/src/net/http/client.go:592
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e000}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e36b0}, 0x170ac65f7e00, {0x170ac65d9680, 0x118})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac699d630 sp=0x170ac699d338 pc=0x38de6bd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e000}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x170ac65f7da0, 0x0, {0x0, ...}, ...})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac699d888 sp=0x170ac699d630 pc=0x38ddecd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e000?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de1a48?}, 0x3?, 0x2022a4d?)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac699d960 sp=0x170ac699d888 pc=0x38dd40b
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194850?}, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x170ac5de1a48, 0x3, 0x4})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac699d9e0 sp=0x170ac699d960 pc=0x38e51e2
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194850}, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x0?, 0x2?, 0x0?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac699da78 sp=0x170ac699d9e0 pc=0x38e4eaf
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac60207c0}, 0x170ac65fa000)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac699dd60 sp=0x170ac699da78 pc=0x391b5c5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac60207c0?}, 0x0?)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac699de88 sp=0x170ac699dd60 pc=0x3929c16
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac699df78 sp=0x170ac699de88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac699dfe0 sp=0x170ac699df78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac699dfe8 sp=0x170ac699dfe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 107 gp=0x170ac61d45a0 m=nil [select]:
    runtime.gopark(0x170ac69b2ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69b29d8?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69b2848 sp=0x170ac69b2828 pc=0x1f38d2e
    runtime.selectgo(0x170ac69b2ac8, 0x170ac69b29d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69b2978 sp=0x170ac69b2848 pc=0x1f152a5
    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e500, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69b2b40 sp=0x170ac69b2978 pc=0x22d92e5
    net/http.(*http2ClientConn).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8603
    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e500, {0x4c?, 0xfa?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69b2c80 sp=0x170ac69b2b40 pc=0x22d50ad
    net/http.(*http2Transport).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:7852
    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e500?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69b2cc0 sp=0x170ac69b2c80 pc=0x22e3a9c
    net/http.(*Transport).roundTrip(0x8175400, 0x170ac699e3c0)
    	/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69b2ea0 sp=0x170ac69b2cc0 pc=0x231037f
    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)
    	/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69b2ec0 sp=0x170ac69b2ea0 pc=0x2321178
    net/http.send(0x170ac699e3c0, {0x59a1ce0, 0x8175400}, {0x170ac69b3138?, 0x1f3bac6?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69b30b0 sp=0x170ac69b2ec0 pc=0x22ad9ab
    net/http.(*Client).send(0x170ac6410c60, 0x170ac699e3c0, {0x4d3aa8?, 0xcd6610?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69b3148 sp=0x170ac69b30b0 pc=0x22ad1d8
    net/http.(*Client).do(0x170ac6410c60, 0x170ac699e3c0)
    	/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69b3338 sp=0x170ac69b3148 pc=0x22af817
    net/http.(*Client).Do(...)
    	/usr/lib/golang/src/net/http/client.go:592
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e1e0}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3a60}, 0x170ac69a0120, {0x170ac65d98c0, 0x111})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69b3630 sp=0x170ac69b3338 pc=0x38de6bd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e1e0}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x170ac69a00c0, 0x0, {0x0, ...}, ...})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69b3888 sp=0x170ac69b3630 pc=0x38ddecd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e1e0?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de2a48?}, 0x3?, 0x2022a4d?)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69b3960 sp=0x170ac69b3888 pc=0x38dd40b
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194bd0?}, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x170ac5de2a48, 0x3, 0x4})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69b39e0 sp=0x170ac69b3960 pc=0x38e51e2
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194bd0}, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x0?, 0x2?, 0x0?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69b3a78 sp=0x170ac69b39e0 pc=0x38e4eaf
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020ec0}, 0x170ac65fa000)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69b3d60 sp=0x170ac69b3a78 pc=0x391b5c5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020ec0?}, 0x0?)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69b3e88 sp=0x170ac69b3d60 pc=0x3929c16
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69b3f78 sp=0x170ac69b3e88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69b3fe0 sp=0x170ac69b3f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69b3fe8 sp=0x170ac69b3fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 106 gp=0x170ac648b680 m=nil [select]:
    runtime.gopark(0x170ac69aaac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69aa9d8?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69aa848 sp=0x170ac69aa828 pc=0x1f38d2e
    runtime.selectgo(0x170ac69aaac8, 0x170ac69aa9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69aa978 sp=0x170ac69aa848 pc=0x1f152a5
    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e280, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69aab40 sp=0x170ac69aa978 pc=0x22d92e5
    net/http.(*http2ClientConn).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8603
    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e280, {0x4c?, 0xfa?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69aac80 sp=0x170ac69aab40 pc=0x22d50ad
    net/http.(*http2Transport).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:7852
    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e280?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69aacc0 sp=0x170ac69aac80 pc=0x22e3a9c
    net/http.(*Transport).roundTrip(0x8175400, 0x170ac699e140)
    	/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69aaea0 sp=0x170ac69aacc0 pc=0x231037f
    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)
    	/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69aaec0 sp=0x170ac69aaea0 pc=0x2321178
    net/http.send(0x170ac699e140, {0x59a1ce0, 0x8175400}, {0x170ac69ab138?, 0x1f3bac6?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69ab0b0 sp=0x170ac69aaec0 pc=0x22ad9ab
    net/http.(*Client).send(0x170ac6410c60, 0x170ac699e140, {0x170ac69ab1e8?, 0x1fdd787?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69ab148 sp=0x170ac69ab0b0 pc=0x22ad1d8
    net/http.(*Client).do(0x170ac6410c60, 0x170ac699e140)
    	/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69ab338 sp=0x170ac69ab148 pc=0x22af817
    net/http.(*Client).Do(...)
    	/usr/lib/golang/src/net/http/client.go:592
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e0f0}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e38c0}, 0x170ac69a0030, {0x170ac65d97a0, 0x10e})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69ab630 sp=0x170ac69ab338 pc=0x38de6bd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e0f0}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x170ac65f7ec0, 0x0, {0x0, ...}, ...})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69ab888 sp=0x170ac69ab630 pc=0x38ddecd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e0f0?}, 0xa8?, {0x170ac5db1a50?, 0x170ac6766a48?}, 0x3?, 0x2022a4d?)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69ab960 sp=0x170ac69ab888 pc=0x38dd40b
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194a80?}, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x170ac6766a48, 0x3, 0x4})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69ab9e0 sp=0x170ac69ab960 pc=0x38e51e2
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194a80}, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x0?, 0x2?, 0x0?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69aba78 sp=0x170ac69ab9e0 pc=0x38e4eaf
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020cc0}, 0x170ac65fa000)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69abd60 sp=0x170ac69aba78 pc=0x391b5c5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac6766ea0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020cc0?}, 0x170ac5f6f3b0?)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69abe88 sp=0x170ac69abd60 pc=0x3929c16
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69abf78 sp=0x170ac69abe88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69abfe0 sp=0x170ac69abf78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69abfe8 sp=0x170ac69abfe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 108 gp=0x170ac61d4780 m=nil [select]:
    runtime.gopark(0x170ac69c0ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69c09d8?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69c0848 sp=0x170ac69c0828 pc=0x1f38d2e
    runtime.selectgo(0x170ac69c0ac8, 0x170ac69c09d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69c0978 sp=0x170ac69c0848 pc=0x1f152a5
    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e780, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69c0b40 sp=0x170ac69c0978 pc=0x22d92e5
    net/http.(*http2ClientConn).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8603
    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e780, {0x4c?, 0xfa?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69c0c80 sp=0x170ac69c0b40 pc=0x22d50ad
    net/http.(*http2Transport).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:7852
    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e780?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69c0cc0 sp=0x170ac69c0c80 pc=0x22e3a9c
    net/http.(*Transport).roundTrip(0x8175400, 0x170ac699e640)
    	/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69c0ea0 sp=0x170ac69c0cc0 pc=0x231037f
    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)
    	/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69c0ec0 sp=0x170ac69c0ea0 pc=0x2321178
    net/http.send(0x170ac699e640, {0x59a1ce0, 0x8175400}, {0x170ac69c1138?, 0x1f3bac6?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69c10b0 sp=0x170ac69c0ec0 pc=0x22ad9ab
    net/http.(*Client).send(0x170ac6410c60, 0x170ac699e640, {0x82bed80?, 0x6401fdd700?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69c1148 sp=0x170ac69c10b0 pc=0x22ad1d8
    net/http.(*Client).do(0x170ac6410c60, 0x170ac699e640)
    	/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69c1338 sp=0x170ac69c1148 pc=0x22af817
    net/http.(*Client).Do(...)
    	/usr/lib/golang/src/net/http/client.go:592
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e280}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3bf0}, 0x170ac69a0210, {0x170ac65d99e0, 0x110})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69c1630 sp=0x170ac69c1338 pc=0x38de6bd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e280}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x170ac69a01b0, 0x0, {0x0, ...}, ...})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69c1888 sp=0x170ac69c1630 pc=0x38ddecd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e280?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de2a48?}, 0x3?, 0x2022a4d?)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69c1960 sp=0x170ac69c1888 pc=0x38dd40b
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194d90?}, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x170ac5de2a48, 0x3, 0x4})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69c19e0 sp=0x170ac69c1960 pc=0x38e51e2
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194d90}, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x0?, 0x2?, 0x0?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69c1a78 sp=0x170ac69c19e0 pc=0x38e4eaf
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6021d80}, 0x170ac65fa000)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69c1d60 sp=0x170ac69c1a78 pc=0x391b5c5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac66e1d6a?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6021d80?}, 0x0?)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69c1e88 sp=0x170ac69c1d60 pc=0x3929c16
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69c1f78 sp=0x170ac69c1e88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69c1fe0 sp=0x170ac69c1f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69c1fe8 sp=0x170ac69c1fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 109 gp=0x170ac61d4960 m=nil [select]:
    runtime.gopark(0x170ac69caac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69ca9d8?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69ca848 sp=0x170ac69ca828 pc=0x1f38d2e
    runtime.selectgo(0x170ac69caac8, 0x170ac69ca9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69ca978 sp=0x170ac69ca848 pc=0x1f152a5
    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699ea00, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69cab40 sp=0x170ac69ca978 pc=0x22d92e5
    net/http.(*http2ClientConn).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8603
    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699ea00, {0x4c?, 0xfa?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69cac80 sp=0x170ac69cab40 pc=0x22d50ad
    net/http.(*http2Transport).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:7852
    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699ea00?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69cacc0 sp=0x170ac69cac80 pc=0x22e3a9c
    net/http.(*Transport).roundTrip(0x8175400, 0x170ac699e8c0)
    	/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69caea0 sp=0x170ac69cacc0 pc=0x231037f
    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)
    	/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69caec0 sp=0x170ac69caea0 pc=0x2321178
    net/http.send(0x170ac699e8c0, {0x59a1ce0, 0x8175400}, {0x170ac69cb138?, 0x1f3bac6?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69cb0b0 sp=0x170ac69caec0 pc=0x22ad9ab
    net/http.(*Client).send(0x170ac6410c60, 0x170ac699e8c0, {0x170ac69cb1e8?, 0x1fdd787?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69cb148 sp=0x170ac69cb0b0 pc=0x22ad1d8
    net/http.(*Client).do(0x170ac6410c60, 0x170ac699e8c0)
    	/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69cb338 sp=0x170ac69cb148 pc=0x22af817
    net/http.(*Client).Do(...)
    	/usr/lib/golang/src/net/http/client.go:592
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e410}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3d80}, 0x170ac69a0300, {0x170ac65d9b00, 0x114})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69cb630 sp=0x170ac69cb338 pc=0x38de6bd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e410}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x170ac69a02a0, 0x0, {0x0, ...}, ...})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69cb888 sp=0x170ac69cb630 pc=0x38ddecd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e410?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de3a48?}, 0x3?, 0x2022a4d?)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69cb960 sp=0x170ac69cb888 pc=0x38dd40b
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194620?}, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x170ac5de3a48, 0x3, 0x4})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69cb9e0 sp=0x170ac69cb960 pc=0x38e51e2
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194620}, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x0?, 0x2?, 0x2700000000000000?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69cba78 sp=0x170ac69cb9e0 pc=0x38e4eaf
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac65f81c0}, 0x170ac65fa000)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69cbd60 sp=0x170ac69cba78 pc=0x391b5c5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac65f81c0?}, 0x0?)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69cbe88 sp=0x170ac69cbd60 pc=0x3929c16
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69cbf78 sp=0x170ac69cbe88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69cbfe0 sp=0x170ac69cbf78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69cbfe8 sp=0x170ac69cbfe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 110 gp=0x170ac61d4b40 m=nil [select]:
    runtime.gopark(0x170ac6927f08?, 0x6?, 0x40?, 0x4b?, 0x170ac6927da4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6927c08 sp=0x170ac6927be8 pc=0x1f38d2e
    runtime.selectgo(0x170ac6927f08, 0x170ac6927d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6927d38 sp=0x170ac6927c08 pc=0x1f152a5
    net/http.(*http2clientStream).writeRequest(0x170ac61a0f00, 0x170ac619fb80, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6927f90 sp=0x170ac6927d38 pc=0x22da6a8
    net/http.(*http2clientStream).doRequest(0x170ac61a0f00, 0x0?, 0x170ac66e2798?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6927fb8 sp=0x170ac6927f90 pc=0x22d99d8
    net/http.(*http2ClientConn).roundTrip.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6927fe0 sp=0x170ac6927fb8 pc=0x22d999f
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6927fe8 sp=0x170ac6927fe0 pc=0x1f40801
    created by net/http.(*http2ClientConn).roundTrip in goroutine 104
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470
    
    goroutine 111 gp=0x170ac61d4d20 m=nil [select]:
    runtime.gopark(0x170ac658df08?, 0x6?, 0x0?, 0x7?, 0x170ac658dda4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac658dc08 sp=0x170ac658dbe8 pc=0x1f38d2e
    runtime.selectgo(0x170ac658df08, 0x170ac658dd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac658dd38 sp=0x170ac658dc08 pc=0x1f152a5
    net/http.(*http2clientStream).writeRequest(0x170ac61a1080, 0x170ac699e000, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac658df90 sp=0x170ac658dd38 pc=0x22da6a8
    net/http.(*http2clientStream).doRequest(0x170ac61a1080, 0x0?, 0x170ac66e2798?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac658dfb8 sp=0x170ac658df90 pc=0x22d99d8
    net/http.(*http2ClientConn).roundTrip.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac658dfe0 sp=0x170ac658dfb8 pc=0x22d999f
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac658dfe8 sp=0x170ac658dfe0 pc=0x1f40801
    created by net/http.(*http2ClientConn).roundTrip in goroutine 105
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470
    
    goroutine 112 gp=0x170ac61d4f00 m=nil [select]:
    runtime.gopark(0x170ac6589f08?, 0x6?, 0x0?, 0x7?, 0x170ac6589da4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6589c08 sp=0x170ac6589be8 pc=0x1f38d2e
    runtime.selectgo(0x170ac6589f08, 0x170ac6589d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6589d38 sp=0x170ac6589c08 pc=0x1f152a5
    net/http.(*http2clientStream).writeRequest(0x170ac61a1200, 0x170ac699e280, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6589f90 sp=0x170ac6589d38 pc=0x22da6a8
    net/http.(*http2clientStream).doRequest(0x170ac61a1200, 0x0?, 0x170ac66e1f98?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6589fb8 sp=0x170ac6589f90 pc=0x22d99d8
    net/http.(*http2ClientConn).roundTrip.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6589fe0 sp=0x170ac6589fb8 pc=0x22d999f
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6589fe8 sp=0x170ac6589fe0 pc=0x1f40801
    created by net/http.(*http2ClientConn).roundTrip in goroutine 106
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470
    
    goroutine 113 gp=0x170ac61d50e0 m=nil [select]:
    runtime.gopark(0x170ac658ff08?, 0x6?, 0x0?, 0x7?, 0x170ac658fda4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac658fc08 sp=0x170ac658fbe8 pc=0x1f38d2e
    runtime.selectgo(0x170ac658ff08, 0x170ac658fd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac658fd38 sp=0x170ac658fc08 pc=0x1f152a5
    net/http.(*http2clientStream).writeRequest(0x170ac61a1380, 0x170ac699e780, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac658ff90 sp=0x170ac658fd38 pc=0x22da6a8
    net/http.(*http2clientStream).doRequest(0x170ac61a1380, 0x0?, 0x170ac66e1f98?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac658ffb8 sp=0x170ac658ff90 pc=0x22d99d8
    net/http.(*http2ClientConn).roundTrip.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac658ffe0 sp=0x170ac658ffb8 pc=0x22d999f
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac658ffe8 sp=0x170ac658ffe0 pc=0x1f40801
    created by net/http.(*http2ClientConn).roundTrip in goroutine 108
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470
    
    goroutine 114 gp=0x170ac61d52c0 m=nil [select]:
    runtime.gopark(0x170ac6923f08?, 0x6?, 0x0?, 0x7?, 0x170ac6923da4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6923c08 sp=0x170ac6923be8 pc=0x1f38d2e
    runtime.selectgo(0x170ac6923f08, 0x170ac6923d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6923d38 sp=0x170ac6923c08 pc=0x1f152a5
    net/http.(*http2clientStream).writeRequest(0x170ac61a1500, 0x170ac699ea00, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6923f90 sp=0x170ac6923d38 pc=0x22da6a8
    net/http.(*http2clientStream).doRequest(0x170ac61a1500, 0x0?, 0x170ac66e1f98?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6923fb8 sp=0x170ac6923f90 pc=0x22d99d8
    net/http.(*http2ClientConn).roundTrip.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6923fe0 sp=0x170ac6923fb8 pc=0x22d999f
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6923fe8 sp=0x170ac6923fe0 pc=0x1f40801
    created by net/http.(*http2ClientConn).roundTrip in goroutine 109
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470
    
    goroutine 115 gp=0x170ac61d54a0 m=nil [select]:
    runtime.gopark(0x170ac6925f08?, 0x6?, 0x38?, 0x5b?, 0x170ac6925da4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6925c08 sp=0x170ac6925be8 pc=0x1f38d2e
    runtime.selectgo(0x170ac6925f08, 0x170ac6925d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6925d38 sp=0x170ac6925c08 pc=0x1f152a5
    net/http.(*http2clientStream).writeRequest(0x170ac61a1680, 0x170ac699e500, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6925f90 sp=0x170ac6925d38 pc=0x22da6a8
    net/http.(*http2clientStream).doRequest(0x170ac61a1680, 0x0?, 0x170ac66e1f98?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6925fb8 sp=0x170ac6925f90 pc=0x22d99d8
    net/http.(*http2ClientConn).roundTrip.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6925fe0 sp=0x170ac6925fb8 pc=0x22d999f
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6925fe8 sp=0x170ac6925fe0 pc=0x1f40801
    created by net/http.(*http2ClientConn).roundTrip in goroutine 107
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470
    
    rax    0x0
    rbx    0x7f7eabfff640
    rcx    0x7f7ef9baa94c
    rdx    0x6
    rdi    0x10a2e
    rsi    0x10a30
    rbp    0x10a30
    rsp    0x7f7eabffe790
    r8     0x7f7eabffe860
    r9     0x7f7ef9cd04e0
    r10    0x8
    r11    0x246
    r12    0x6
    r13    0x7f7eabffeb60
    r14    0x170ac5c1f0e0
    r15    0xffffffffffffffff
    rip    0x7f7ef9baa94c
    rflags 0x246
    cs     0x33
    fs     0x0
    gs     0x0
    
      query-infra-both: command failed: exit status 2
    output: runtime/cgo: pthread_create failed: Resource temporarily unavailable
    SIGABRT: abort
    PC=0x7f5e9c16994c m=2 sigcode=18446744073709551610
    
    goroutine 0 gp=0x280029a48b40 m=2 mp=0x280029abb008 [idle]:
    runtime: g 0 gp=0x280029a48b40: unknown pc 0x7f5e9c16994c
    stack: frame={sp:0x7f5e554a7740, fp:0x0} stack=[0x7f5e54ca81e0,0x7f5e554a7de0)
                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef
    00007f5e554a7640: 00000000 01eb64e5  00000000 00000000  .d..............
                      ^ <internal/runtime/syscall/linux.EpollWait+0x45>
    00007f5e554a7650: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7660: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7670: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7680: 00007f5e 554a76d4  00007f5e 554a7d18  .vJU^....}JU^...
    00007f5e554a7690: 00000000 01efa894  00000000 00000000  ................
                      ^ <runtime.netpoll+0xd4>
    00007f5e554a76a0: 00007f5e 554a76d4  00000000 00000000  .vJU^...........
    00007f5e554a76b0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a76c0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a76d0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a76e0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a76f0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7700: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7710: 00000000 00000000  00007f5e 9c176fa9  .........o..^...
    00007f5e554a7720: 00000000 00000000  00000000 00000011  ................
    00007f5e554a7730: 00000000 00000130  00007f5e 9c16993e  0.......>...^...
    00007f5e554a7740: 00000000 00000120  00000000 00000000   ...............
                      ^ SP
    00007f5e554a7750: 00000000 00000013  00000000 00000000  ................
    00007f5e554a7760: 00000000 00000000  00007f5e 9c1eccff  ............^...
    00007f5e554a7770: 00007f5e 327fc910  00007f5e 9c1eccb1  ...2^.......^...
    00007f5e554a7780: 00007f5e 327fc640  00007f5e 554a7a50  @..2^...PzJU^...
    00007f5e554a7790: 00007f5e 554a78be  00007f5e 554a78bf  .xJU^....xJU^...
    00007f5e554a77a0: 00007f5e 327fc640  00007f5e 9c1677f5  @..2^....w..^...
    00007f5e554a77b0: 00007f5e 50000020  00000000 00000120   ..P^... .......
    00007f5e554a77c0: 00000000 003d0f00  5a23f32c fe4c8400  ..=.......L.,.#Z
    00007f5e554a77d0: 00007f5e 554a8640  00000000 00000006  @.JU^...........
    00007f5e554a77e0: 00000000 0bac42a0  00007f5e 554a7b10  .B.......{JU^...
    00007f5e554a77f0: 00002800 29a48b40  00007f5e 9c11c646  @..).(..F...^...
    00007f5e554a7800: 00007f5e 9c2d9e90  00007f5e 9c1067f3  ..-.^....g..^...
    00007f5e554a7810: 00000000 00000020  5a23f32c fe4c8400   .........L.,.#Z
    00007f5e554a7820: 00007f5e 31ffc000  00007f5e 9c1dbac4  ...1^.......^...
    00007f5e554a7830: 00000000 00000000  00000000 0000000d  ................
    runtime: g 0 gp=0x280029a48b40: unknown pc 0x7f5e9c16994c
    stack: frame={sp:0x7f5e554a7740, fp:0x0} stack=[0x7f5e54ca81e0,0x7f5e554a7de0)
                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef
    00007f5e554a7640: 00000000 01eb64e5  00000000 00000000  .d..............
                      ^ <internal/runtime/syscall/linux.EpollWait+0x45>
    00007f5e554a7650: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7660: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7670: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7680: 00007f5e 554a76d4  00007f5e 554a7d18  .vJU^....}JU^...
    00007f5e554a7690: 00000000 01efa894  00000000 00000000  ................
                      ^ <runtime.netpoll+0xd4>
    00007f5e554a76a0: 00007f5e 554a76d4  00000000 00000000  .vJU^...........
    00007f5e554a76b0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a76c0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a76d0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a76e0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a76f0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7700: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7710: 00000000 00000000  00007f5e 9c176fa9  .........o..^...
    00007f5e554a7720: 00000000 00000000  00000000 00000011  ................
    00007f5e554a7730: 00000000 00000130  00007f5e 9c16993e  0.......>...^...
    00007f5e554a7740: 00000000 00000120  00000000 00000000   ...............
                      ^ SP
    00007f5e554a7750: 00000000 00000013  00000000 00000000  ................
    00007f5e554a7760: 00000000 00000000  00007f5e 9c1eccff  ............^...
    00007f5e554a7770: 00007f5e 327fc910  00007f5e 9c1eccb1  ...2^.......^...
    00007f5e554a7780: 00007f5e 327fc640  00007f5e 554a7a50  @..2^...PzJU^...
    00007f5e554a7790: 00007f5e 554a78be  00007f5e 554a78bf  .xJU^....xJU^...
    00007f5e554a77a0: 00007f5e 327fc640  00007f5e 9c1677f5  @..2^....w..^...
    00007f5e554a77b0: 00007f5e 50000020  00000000 00000120   ..P^... .......
    00007f5e554a77c0: 00000000 003d0f00  5a23f32c fe4c8400  ..=.......L.,.#Z
    00007f5e554a77d0: 00007f5e 554a8640  00000000 00000006  @.JU^...........
    00007f5e554a77e0: 00000000 0bac42a0  00007f5e 554a7b10  .B.......{JU^...
    00007f5e554a77f0: 00002800 29a48b40  00007f5e 9c11c646  @..).(..F...^...
    00007f5e554a7800: 00007f5e 9c2d9e90  00007f5e 9c1067f3  ..-.^....g..^...
    00007f5e554a7810: 00000000 00000020  5a23f32c fe4c8400   .........L.,.#Z
    00007f5e554a7820: 00007f5e 31ffc000  00007f5e 9c1dbac4  ...1^.......^...
    00007f5e554a7830: 00000000 00000000  00000000 0000000d  ................
    
    goroutine 1 gp=0x280029a481e0 m=nil [sync.WaitGroup.Wait]:
    runtime.gopark(0x28002a5927b8?, 0x2022a4d?, 0x30?, 0xe9?, 0x28002a5927e8?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a5927a0 sp=0x28002a592780 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x280029ae34d0, 0x0, 0x1, 0x0, 0x19)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a592808 sp=0x28002a5927a0 pc=0x1f162b2
    sync.runtime_SemacquireWaitGroup(0x28002a83b578?, 0xe0?)
    	/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x28002a592840 sp=0x28002a592808 pc=0x1f3a66e
    sync.(*WaitGroup).Wait(0x280029ae34c8)
    	/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x28002a592868 sp=0x28002a592840 pc=0x1f4f445
    golang.org/x/sync/errgroup.(*Group).Wait(0x280029ae34c0)
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x28002a592888 sp=0x28002a592868 pc=0x362507e
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0}, {0x28002a977b50, 0x1, 0x1})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x28002a592948 sp=0x28002a592888 pc=0x3927e05
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).gatherInfraLogs(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:587 +0x867 fp=0x28002a592d30 sp=0x28002a592948 pc=0x3927727
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:333 +0x533 fp=0x28002a593908 sp=0x28002a592d30 pc=0x3921bf3
    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedInfraQueryOptions).RunInfra(0x28002a11e1e0, {0x59f0b08, 0x28002a1873b0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:83 +0x71f fp=0x28002a593b30 sp=0x28002a593908 pc=0x393887f
    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawInfraQueryOptions).Run(0x0?, {0x59f0b08, 0x28002a1873b0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_options.go:109 +0x54 fp=0x28002a593b58 sp=0x28002a593b30 pc=0x3939134
    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryInfraCommand.func1(0x28002a53d508?, {0x280029ca3b00?, 0x4?, 0x553b0e2?})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:45 +0x28 fp=0x28002a593b80 sp=0x28002a593b58 pc=0x3938128
    github.com/spf13/cobra.(*Command).execute(0x28002a53d508, {0x280029ab8120, 0xc, 0xc})
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x28002a593d28 sp=0x28002a593b80 pc=0x212e614
    github.com/spf13/cobra.(*Command).ExecuteC(0x280029ec9b08)
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x28002a593e18 sp=0x28002a593d28 pc=0x212efa5
    github.com/spf13/cobra.(*Command).Execute(...)
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071
    github.com/spf13/cobra.(*Command).ExecuteContext(...)
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064
    main.main()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x28002a593f48 sp=0x28002a593e18 pc=0x47ddd6a
    runtime.main()
    	/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x28002a593fe0 sp=0x28002a593f48 pc=0x1f01975
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a593fe8 sp=0x28002a593fe0 pc=0x1f40801
    
    goroutine 2 gp=0x280029a48d20 m=nil [force gc (idle)]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b08fa8 sp=0x280029b08f88 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.forcegchelper()
    	/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x280029b08fe0 sp=0x280029b08fa8 pc=0x1f01c93
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b08fe8 sp=0x280029b08fe0 pc=0x1f40801
    created by runtime.init.7 in goroutine 1
    	/usr/lib/golang/src/runtime/proc.go:363 +0x1a
    
    goroutine 3 gp=0x280029a492c0 m=nil [GC sweep wait]:
    runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b09788 sp=0x280029b09768 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.bgsweep(0x280029b30000)
    	/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x280029b097c8 sp=0x280029b09788 pc=0x1ee9f11
    runtime.gcenable.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x280029b097e0 sp=0x280029b097c8 pc=0x1edadd7
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b097e8 sp=0x280029b097e0 pc=0x1f40801
    created by runtime.gcenable in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:214 +0x66
    
    goroutine 4 gp=0x280029a494a0 m=nil [GC scavenge wait]:
    runtime.gopark(0x10000?, 0x5981cd8?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b09f78 sp=0x280029b09f58 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.(*scavengerState).park(0x82979c0)
    	/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x280029b09fa8 sp=0x280029b09f78 pc=0x1ee77e9
    runtime.bgscavenge(0x280029b30000)
    	/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x280029b09fc8 sp=0x280029b09fa8 pc=0x1ee7d99
    runtime.gcenable.gowrap2()
    	/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x280029b09fe0 sp=0x280029b09fc8 pc=0x1edad97
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b09fe8 sp=0x280029b09fe0 pc=0x1f40801
    created by runtime.gcenable in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:215 +0xa5
    
    goroutine 5 gp=0x280029a49a40 m=nil [GOMAXPROCS updater (idle)]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0a788 sp=0x280029b0a768 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.updateMaxProcsGoroutine()
    	/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x280029b0a7e0 sp=0x280029b0a788 pc=0x1f10267
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0a7e8 sp=0x280029b0a7e0 pc=0x1f40801
    created by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1
    	/usr/lib/golang/src/runtime/proc.go:7083 +0x37
    
    goroutine 6 gp=0x280029b481e0 m=nil [finalizer wait]:
    runtime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x280029b08601?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b08620 sp=0x280029b08600 pc=0x1f38d2e
    runtime.runFinalizers()
    	/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x280029b087e0 sp=0x280029b08620 pc=0x1ed9cc7
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b087e8 sp=0x280029b087e0 pc=0x1f40801
    created by runtime.createfing in goroutine 1
    	/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d
    
    goroutine 7 gp=0x280029d3fc20 m=nil [cleanup wait]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0af68 sp=0x280029b0af48 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.(*cleanupQueue).dequeue(0x82980e0)
    	/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x280029b0afa0 sp=0x280029b0af68 pc=0x1ed6074
    runtime.runCleanups()
    	/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x280029b0afe0 sp=0x280029b0afa0 pc=0x1ed66e5
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0afe8 sp=0x280029b0afe0 pc=0x1f40801
    created by runtime.(*cleanupQueue).createGs in goroutine 1
    	/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5
    
    goroutine 8 gp=0x280029ee6000 m=nil [GC worker (idle)]:
    runtime.gopark(0x125c86ae32d7?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0b740 sp=0x280029b0b720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x280029ad7ab0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x280029b0b7c8 sp=0x280029b0b740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b0b7e0 sp=0x280029b0b7c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0b7e8 sp=0x280029b0b7e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 18 gp=0x280029b843c0 m=nil [GC worker (idle)]:
    runtime.gopark(0x125c662e5759?, 0x3?, 0xc2?, 0x10?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b04740 sp=0x280029b04720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x280029ad7ab0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x280029b047c8 sp=0x280029b04740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b047e0 sp=0x280029b047c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b047e8 sp=0x280029b047e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 9 gp=0x280029ee61e0 m=nil [semacquire]:
    runtime.gopark(0x6e2b22?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029b0bc96?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0be38 sp=0x280029b0be18 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029b0bea0 sp=0x280029b0be38 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029b0bf40 sp=0x280029b0bea0 pc=0x1edbb12
    runtime.gcBgMarkWorker(0x280029ad7ab0)
    	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029b0bfc8 sp=0x280029b0bf40 pc=0x1edda88
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b0bfe0 sp=0x280029b0bfc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0bfe8 sp=0x280029b0bfe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 10 gp=0x280029ee63c0 m=nil [semacquire]:
    runtime.gopark(0x1ec6f5e?, 0x1f3ec12?, 0x80?, 0x51?, 0x1ec70df?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eec638 sp=0x280029eec618 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eec6a0 sp=0x280029eec638 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eec740 sp=0x280029eec6a0 pc=0x1edbb12
    runtime.gcBgMarkWorker(0x280029ad7ab0)
    	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eec7c8 sp=0x280029eec740 pc=0x1edda88
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eec7e0 sp=0x280029eec7c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eec7e8 sp=0x280029eec7e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 11 gp=0x280029ee65a0 m=nil [semacquire]:
    runtime.gopark(0x54b522?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029eecc96?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eece38 sp=0x280029eece18 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eecea0 sp=0x280029eece38 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eecf40 sp=0x280029eecea0 pc=0x1edbb12
    runtime.gcBgMarkWorker(0x280029ad7ab0)
    	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eecfc8 sp=0x280029eecf40 pc=0x1edda88
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eecfe0 sp=0x280029eecfc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eecfe8 sp=0x280029eecfe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 34 gp=0x280029f04000 m=nil [semacquire]:
    runtime.gopark(0x2d7122?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029ee8496?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029ee8638 sp=0x280029ee8618 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029ee86a0 sp=0x280029ee8638 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029ee8740 sp=0x280029ee86a0 pc=0x1edbb12
    runtime.gcBgMarkWorker(0x280029ad7ab0)
    	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029ee87c8 sp=0x280029ee8740 pc=0x1edda88
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029ee87e0 sp=0x280029ee87c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029ee87e8 sp=0x280029ee87e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 19 gp=0x280029b845a0 m=4 mp=0x280029b34008 [GC mark termination]:
    runtime.systemstack_switch()
    	/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x280029b04ea0 sp=0x280029b04e90 pc=0x1f3ebc8
    runtime.forEachP(...)
    	/usr/lib/golang/src/runtime/proc.go:2112
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x280029b04f40 sp=0x280029b04ea0 pc=0x1edbc34
    runtime.gcBgMarkWorker(0x280029ad7ab0)
    	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029b04fc8 sp=0x280029b04f40 pc=0x1edda88
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b04fe0 sp=0x280029b04fc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b04fe8 sp=0x280029b04fe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 12 gp=0x280029ee6780 m=nil [semacquire]:
    runtime.gopark(0x3f9e22?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029eed496?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eed638 sp=0x280029eed618 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eed6a0 sp=0x280029eed638 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eed740 sp=0x280029eed6a0 pc=0x1edbb12
    runtime.gcBgMarkWorker(0x280029ad7ab0)
    	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eed7c8 sp=0x280029eed740 pc=0x1edda88
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eed7e0 sp=0x280029eed7c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eed7e8 sp=0x280029eed7e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 35 gp=0x280029ee74a0 m=nil [select, locked to thread]:
    runtime.gopark(0x280029eef7a8?, 0x2?, 0x1?, 0x74?, 0x280029eef794?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eef628 sp=0x280029eef608 pc=0x1f38d2e
    runtime.selectgo(0x280029eef7a8, 0x280029eef790, 0x0?, 0x0, 0x0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x280029eef758 sp=0x280029eef628 pc=0x1f152a5
    runtime.ensureSigM.func1()
    	/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x280029eef7e0 sp=0x280029eef758 pc=0x1f328e8
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eef7e8 sp=0x280029eef7e0 pc=0x1f40801
    created by runtime.ensureSigM in goroutine 1
    	/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5
    
    goroutine 50 gp=0x280029f84960 m=9 mp=0x28002a000008 [syscall]:
    runtime.notetsleepg(0x82bdea0, 0xffffffffffffffff)
    	/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x280029ee9fa0 sp=0x280029ee9f78 pc=0x1ece509
    os/signal.signal_recv()
    	/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x280029ee9fc0 sp=0x280029ee9fa0 pc=0x1f3b298
    os/signal.loop()
    	/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x280029ee9fe0 sp=0x280029ee9fc0 pc=0x2030553
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029ee9fe8 sp=0x280029ee9fe0 pc=0x1f40801
    created by os/signal.Notify.func1.1 in goroutine 1
    	/usr/lib/golang/src/os/signal/signal.go:152 +0x1f
    
    goroutine 51 gp=0x280029f84b40 m=nil [select]:
    runtime.gopark(0x280029eea798?, 0x2?, 0x80?, 0xa3?, 0x280029eea78c?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eea628 sp=0x280029eea608 pc=0x1f38d2e
    runtime.selectgo(0x280029eea798, 0x280029eea788, 0x0?, 0x0, 0x0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x280029eea758 sp=0x280029eea628 pc=0x1f152a5
    os/signal.NotifyContext.func1()
    	/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x280029eea7e0 sp=0x280029eea758 pc=0x202ff9c
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eea7e8 sp=0x280029eea7e0 pc=0x1f40801
    created by os/signal.NotifyContext in goroutine 1
    	/usr/lib/golang/src/os/signal/signal.go:291 +0x158
    
    goroutine 137 gp=0x28002a60a000 m=nil [sync.WaitGroup.Wait]:
    runtime.gopark(0x28002a603e00?, 0x1ed0765?, 0x80?, 0xea?, 0x7f5e54234a40?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a603da0 sp=0x28002a603d80 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x280029ae3b10, 0x0, 0x1, 0x0, 0x19)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a603e08 sp=0x28002a603da0 pc=0x1f162b2
    sync.runtime_SemacquireWaitGroup(0x28002a83b590?, 0x0?)
    	/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x28002a603e40 sp=0x28002a603e08 pc=0x1f3a66e
    sync.(*WaitGroup).Wait(0x280029ae3b08)
    	/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x28002a603e68 sp=0x28002a603e40 pc=0x1f4f445
    golang.org/x/sync/errgroup.(*Group).Wait(0x280029ae3b00)
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x28002a603e88 sp=0x28002a603e68 pc=0x362507e
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x280029d80e80, {0x59f0b40, 0x280029beca00}, {0x28002a977b50, 0x1, 0x38d788c?}, 0x28002a830c40)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x28002a603f00 sp=0x28002a603e88 pc=0x392976a
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x28002a603f78 sp=0x28002a603f00 pc=0x392808e
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a603fe0 sp=0x28002a603f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a603fe8 sp=0x28002a603fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 38 gp=0x28002a60b0e0 m=nil [IO wait]:
    runtime.gopark(0x28002a7c22c0?, 0x280029aee7f8?, 0x40?, 0x77?, 0xb?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b197b8 sp=0x280029b19798 pc=0x1f38d2e
    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)
    	/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x280029b197f0 sp=0x280029b197b8 pc=0x1ef9e57
    internal/poll.runtime_pollWait(0x7f5e554b6c00, 0x72)
    	/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x280029b19810 sp=0x280029b197f0 pc=0x1f375c5
    internal/poll.(*pollDesc).wait(0x280029bf6400?, 0x28002a90c000?, 0x0)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x280029b19838 sp=0x280029b19810 pc=0x1fbd167
    internal/poll.(*pollDesc).waitRead(...)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89
    internal/poll.(*FD).Read(0x280029bf6400, {0x28002a90c000, 0x3500, 0x3500})
    	/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x280029b198d0 sp=0x280029b19838 pc=0x1fbe4ce
    net.(*netFD).Read(0x280029bf6400, {0x28002a90c000?, 0x28002a90c000?, 0x5?})
    	/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x280029b19918 sp=0x280029b198d0 pc=0x20798c5
    net.(*conn).Read(0x280029a46378, {0x28002a90c000?, 0x7f5e554a9828?, 0x7f5e9c0b1e00?})
    	/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x280029b19960 sp=0x280029b19918 pc=0x208b7a5
    crypto/tls.(*atLeastReader).Read(0x28002aa26ba0, {0x28002a90c000?, 0x28002a0d8658?, 0x1efcc28?})
    	/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x280029b199a8 sp=0x280029b19960 pc=0x2222ddb
    bytes.(*Buffer).ReadFrom(0x28002a0d8628, {0x59a3680, 0x28002aa26ba0})
    	/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x280029b19a00 sp=0x280029b199a8 pc=0x1fe9258
    crypto/tls.(*Conn).readFromUntil(0x28002a0d8388, {0x59a32c0, 0x280029a46378}, 0x222764c?)
    	/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x280029b19a38 sp=0x280029b19a00 pc=0x2222fbe
    crypto/tls.(*Conn).readRecordOrCCS(0x28002a0d8388, 0x0)
    	/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x280029b19c18 sp=0x280029b19a38 pc=0x222009b
    crypto/tls.(*Conn).readRecord(...)
    	/usr/lib/golang/src/crypto/tls/conn.go:588
    crypto/tls.(*Conn).Read(0x28002a0d8388, {0x280029fed000, 0x1000, 0x22c34e0?})
    	/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x280029b19c80 sp=0x280029b19c18 pc=0x22267c5
    bufio.(*Reader).Read(0x28002a7b89c0, {0x28002a7c8584, 0x9, 0x22e100e?})
    	/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0x280029b19cb8 sp=0x280029b19c80 pc=0x20a71f7
    io.ReadAtLeast({0x59a1260, 0x28002a7b89c0}, {0x28002a7c8584, 0x9, 0x9}, 0x9)
    	/usr/lib/golang/src/io/io.go:335 +0x8e fp=0x280029b19d00 sp=0x280029b19cb8 pc=0x1fb17ee
    io.ReadFull(...)
    	/usr/lib/golang/src/io/io.go:354
    net/http.http2readFrameHeader({0x28002a7c8584, 0x9, 0x280000000015?}, {0x59a1260?, 0x28002a7b89c0?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:1805 +0x65 fp=0x280029b19d50 sp=0x280029b19d00 pc=0x22b9e25
    net/http.(*http2Framer).ReadFrameHeader(0x28002a7c8540)
    	/usr/lib/golang/src/net/http/h2_bundle.go:2071 +0x6b fp=0x280029b19dd8 sp=0x280029b19d50 pc=0x22ba9eb
    net/http.(*http2Framer).ReadFrame(0x28002a7c8540)
    	/usr/lib/golang/src/net/http/h2_bundle.go:2130 +0x18 fp=0x280029b19e18 sp=0x280029b19dd8 pc=0x22bb1b8
    net/http.(*http2clientConnReadLoop).run(0x280029b19fa8)
    	/usr/lib/golang/src/net/http/h2_bundle.go:9550 +0xca fp=0x280029b19f60 sp=0x280029b19e18 pc=0x22de14a
    net/http.(*http2ClientConn).readLoop(0x280029bc01c0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x280029b19fc8 sp=0x280029b19f60 pc=0x22dd532
    net/http.(*http2Transport).newClientConn.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x280029b19fe0 sp=0x280029b19fc8 pc=0x22d7057
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b19fe8 sp=0x280029b19fe0 pc=0x1f40801
    created by net/http.(*http2Transport).newClientConn in goroutine 37
    	/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5
    
    goroutine 69 gp=0x28002a60b4a0 m=nil [IO wait]:
    runtime.gopark(0x28002a50c560?, 0x1edb176?, 0x80?, 0xed?, 0xb?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a606808 sp=0x28002a6067e8 pc=0x1f38d2e
    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)
    	/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x28002a606840 sp=0x28002a606808 pc=0x1ef9e57
    internal/poll.runtime_pollWait(0x7f5e554b6a00, 0x72)
    	/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x28002a606860 sp=0x28002a606840 pc=0x1f375c5
    internal/poll.(*pollDesc).wait(0x280029bf6c80?, 0x280029a5a000?, 0x0)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x28002a606888 sp=0x28002a606860 pc=0x1fbd167
    internal/poll.(*pollDesc).waitRead(...)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89
    internal/poll.(*FD).Read(0x280029bf6c80, {0x280029a5a000, 0x2500, 0x2500})
    	/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x28002a606920 sp=0x28002a606888 pc=0x1fbe4ce
    net.(*netFD).Read(0x280029bf6c80, {0x280029a5a000?, 0x28002a60690a?, 0x1edb176?})
    	/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x28002a606968 sp=0x28002a606920 pc=0x20798c5
    net.(*conn).Read(0x280029a463c0, {0x280029a5a000?, 0x7f5e5423eae0?, 0x7f5e9c09dde0?})
    	/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x28002a6069b0 sp=0x28002a606968 pc=0x208b7a5
    crypto/tls.(*atLeastReader).Read(0x28002a628318, {0x280029a5a000?, 0x28002a606a98?, 0x1efcc28?})
    	/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x28002a6069f8 sp=0x28002a6069b0 pc=0x2222ddb
    bytes.(*Buffer).ReadFrom(0x280029ba9ea8, {0x59a3680, 0x28002a628318})
    	/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x28002a606a50 sp=0x28002a6069f8 pc=0x1fe9258
    crypto/tls.(*Conn).readFromUntil(0x280029ba9c08, {0x59a32c0, 0x280029a463c0}, 0x222764c?)
    	/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x28002a606a88 sp=0x28002a606a50 pc=0x2222fbe
    crypto/tls.(*Conn).readRecordOrCCS(0x280029ba9c08, 0x0)
    	/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x28002a606c68 sp=0x28002a606a88 pc=0x222009b
    crypto/tls.(*Conn).readRecord(...)
    	/usr/lib/golang/src/crypto/tls/conn.go:588
    crypto/tls.(*Conn).Read(0x280029ba9c08, {0x28002a24a000, 0x1000, 0x4e55540?})
    	/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x28002a606cd0 sp=0x28002a606c68 pc=0x22267c5
    net/http.(*persistConn).Read(0x28002a0ad400, {0x28002a24a000?, 0x59a0ea0?, 0x81260e0?})
    	/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x28002a606d30 sp=0x28002a606cd0 pc=0x2319d07
    bufio.(*Reader).fill(0x28002a8130e0)
    	/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x28002a606d68 sp=0x28002a606d30 pc=0x20a6c63
    bufio.(*Reader).Peek(0x28002a8130e0, 0x1)
    	/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x28002a606d80 sp=0x28002a606d68 pc=0x20a6df3
    net/http.(*persistConn).readLoop(0x28002a0ad400)
    	/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x28002a606fc8 sp=0x28002a606d80 pc=0x231a812
    net/http.(*Transport).dialConn.gowrap2()
    	/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x28002a606fe0 sp=0x28002a606fc8 pc=0x2319197
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a606fe8 sp=0x28002a606fe0 pc=0x1f40801
    created by net/http.(*Transport).dialConn in goroutine 55
    	/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac
    
    goroutine 70 gp=0x28002a60bc20 m=nil [select]:
    runtime.gopark(0x28002a613f48?, 0x2?, 0x0?, 0xb6?, 0x28002a613ef4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a708d88 sp=0x28002a708d68 pc=0x1f38d2e
    runtime.selectgo(0x28002a708f48, 0x28002a613ef0, 0x280029c93a80?, 0x0, 0x28002a4f7c80?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a708eb8 sp=0x28002a708d88 pc=0x1f152a5
    net/http.(*persistConn).writeLoop(0x28002a0ad400)
    	/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x28002a708fc8 sp=0x28002a708eb8 pc=0x231c426
    net/http.(*Transport).dialConn.gowrap3()
    	/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x28002a708fe0 sp=0x28002a708fc8 pc=0x2319157
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a708fe8 sp=0x28002a708fe0 pc=0x1f40801
    created by net/http.(*Transport).dialConn in goroutine 55
    	/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe
    
    goroutine 138 gp=0x28002a84a780 m=nil [semacquire]:
    runtime.gopark(0x1?, 0x1f78fa5?, 0xe0?, 0x75?, 0x2?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a2ca910 sp=0x28002a2ca8f0 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a2ca978 sp=0x28002a2ca910 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x28002a2caa18 sp=0x28002a2ca978 pc=0x1edbb12
    runtime.gcAssistAlloc(0x28002a84a780)
    	/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x28002a2caaa8 sp=0x28002a2caa18 pc=0x1ee06b6
    runtime.deductAssistCredit(0x28002a2cac70?)
    	/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x28002a2caac8 sp=0x28002a2caaa8 pc=0x1ed10a6
    runtime.mallocgc(0x10, 0x4af39e0, 0x1)
    	/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x28002a2caaf8 sp=0x28002a2caac8 pc=0x1f35e07
    runtime.convTstring({0x28002a6a7728, 0x14})
    	/usr/lib/golang/src/runtime/iface.go:431 +0x45 fp=0x28002a2cab20 sp=0x28002a2caaf8 pc=0x1f35865
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x280029beca00}, 0x28002a830cb0, 0x280029f636e0)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:297 +0x394 fp=0x28002a2caf48 sp=0x28002a2cab20 pc=0x3920674
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x28002a2caf78 sp=0x28002a2caf48 pc=0x392800c
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a2cafe0 sp=0x28002a2caf78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a2cafe8 sp=0x28002a2cafe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 139 gp=0x28002a84a960 m=nil [select]:
    runtime.gopark(0x28002a4bdd00?, 0x2?, 0x1?, 0xc9?, 0x28002a4bdc74?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a4bdac8 sp=0x28002a4bdaa8 pc=0x1f38d2e
    runtime.selectgo(0x28002a4bdd00, 0x28002a4bdc70, 0x28002a3e8a50?, 0x0, 0x280029f827c8?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a4bdbf8 sp=0x28002a4bdac8 pc=0x1f152a5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x49424d554554534c?, {0x59f0b40, 0x280029beca00}, 0x28002a830c40, 0x28002a830cb0)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x28002a4bdf10 sp=0x28002a4bdbf8 pc=0x392836e
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x28002a4bdf78 sp=0x28002a4bdf10 pc=0x3927f3d
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a4bdfe0 sp=0x28002a4bdf78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a4bdfe8 sp=0x28002a4bdfe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 140 gp=0x28002a84ab40 m=nil [semacquire]:
    runtime.gopark(0x1f6aa32?, 0x28002a34b418?, 0xc5?, 0xbd?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a34b3c0 sp=0x28002a34b3a0 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a34b428 sp=0x28002a34b3c0 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x28002a34b4c8 sp=0x28002a34b428 pc=0x1edbb12
    runtime.gcAssistAlloc(0x28002a84ab40)
    	/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x28002a34b558 sp=0x28002a34b4c8 pc=0x1ee06b6
    runtime.deductAssistCredit(0x20?)
    	/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x28002a34b578 sp=0x28002a34b558 pc=0x1ed10a6
    runtime.mallocgc(0x1000, 0x4af32a0, 0x1)
    	/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x28002a34b5a8 sp=0x28002a34b578 pc=0x1f35e07
    runtime.makeslice(0x28002a34b668?, 0x1f8?, 0x1ff?)
    	/usr/lib/golang/src/runtime/slice.go:117 +0x49 fp=0x28002a34b5d0 sp=0x28002a34b5a8 pc=0x1f3b489
    bufio.NewWriterSize(...)
    	/usr/lib/golang/src/bufio/bufio.go:599
    bufio.NewWriter(...)
    	/usr/lib/golang/src/bufio/bufio.go:608
    encoding/csv.NewWriter(...)
    	/usr/lib/golang/src/encoding/csv/writer.go:42
    github.com/Azure/azure-kusto-go/azkustodata/query.(*row).String(0x812ab90?)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/row_impl.go:100 +0x18e fp=0x28002a34b6e8 sp=0x28002a34b5d0 pc=0x38cf5ee
    fmt.(*pp).handleMethods(0x28002a24c0d0, 0x2a34b8f0?)
    	/usr/lib/golang/src/fmt/print.go:674 +0x2f4 fp=0x28002a34b8a8 sp=0x28002a34b6e8 pc=0x1fdd1d4
    fmt.(*pp).printArg(0x28002a24c0d0, {0x549a3a0, 0x28002a7c7c80}, 0x76)
    	/usr/lib/golang/src/fmt/print.go:757 +0x42e fp=0x28002a34b938 sp=0x28002a34b8a8 pc=0x1fdd9ee
    fmt.(*pp).doPrintf(0x28002a24c0d0, {0x553a400, 0x2}, {0x28002a34bd08, 0x1, 0x1})
    	/usr/lib/golang/src/fmt/print.go:1075 +0x3ec fp=0x28002a34ba20 sp=0x28002a34b938 pc=0x1fe04ec
    fmt.Sprintf({0x553a400, 0x2}, {0x28002a34bd08, 0x1, 0x1})
    	/usr/lib/golang/src/fmt/print.go:239 +0x53 fp=0x28002a34ba78 sp=0x28002a34ba20 pc=0x1fda6b3
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x280029e454e0, {0x59f0b40, 0x280029beca50}, {0x5a00300, 0x280029ae2040}, 0x28002a830c40)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:160 +0x8d8 fp=0x28002a34bd60 sp=0x28002a34ba78 pc=0x391bad8
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x2d20414320746f6f?, {0x59f0b40?, 0x280029beca50?}, {0x5a00300?, 0x280029ae2040?}, 0x43437a4b4349494d?)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x28002a34be88 sp=0x28002a34bd60 pc=0x3929c16
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x28002a34bf78 sp=0x28002a34be88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a34bfe0 sp=0x28002a34bf78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a34bfe8 sp=0x28002a34bfe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 137
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 142 gp=0x28002a84af00 m=nil [select]:
    runtime.gopark(0x28002a333de0?, 0x2?, 0x8?, 0xb8?, 0x28002a333dbc?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a333c58 sp=0x28002a333c38 pc=0x1f38d2e
    runtime.selectgo(0x28002a333de0, 0x28002a333db8, 0x280029afc008?, 0x1, 0x1f3665f?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a333d88 sp=0x28002a333c58 pc=0x1f152a5
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.(*iterativeDataset).sendTable(0x280029becf00, {0x5a19be0, 0x28002ab876c0})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:378 +0xf3 fp=0x28002a333e10 sp=0x28002a333d88 pc=0x38d8d33
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.handleDataTable(0x280029becf00, 0x28002a3b2b40)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:324 +0x2c6 fp=0x28002a333ec8 sp=0x28002a333e10 pc=0x38d8686
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x280029becf00)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:167 +0x148 fp=0x28002a333f30 sp=0x28002a333ec8 pc=0x38d7b48
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x280029becf00, 0x28002a977da0)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x28002a333fc0 sp=0x28002a333f30 pc=0x38d788c
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x28002a333fe0 sp=0x28002a333fc0 pc=0x38d751b
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a333fe8 sp=0x28002a333fe0 pc=0x1f40801
    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 140
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc
    
    rax    0x0
    rbx    0x7f5e554a8640
    rcx    0x7f5e9c16994c
    rdx    0x6
    rdi    0x10a66
    rsi    0x10a67
    rbp    0x10a67
    rsp    0x7f5e554a7740
    r8     0x7f5e554a7810
    r9     0x7f5e9c28f4e0
    r10    0x8
    r11    0x246
    r12    0x6
    r13    0x7f5e554a7b10
    r14    0x280029a48b40
    r15    0xffffffffffffffff
    rip    0x7f5e9c16994c
    rflags 0x246
    cs     0x33
    fs     0x0
    gs     0x0
    
    {
        s: "must-gather CLI test failures:\n  query-basic: command failed: exit status 1\noutput: \x1b[37m[21:33:14.493]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104\"\n}\x1b[0m\n\x1b[37m[21:33:23.438]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:21.8572874Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\\\", ActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", SubActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \\u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\\\"}, IsPermanent:false})\",\n  \"name\": \"detailedInfraOrchestrationLogs\"\n}\x1b[0m\n\x1b[37m[21:33:23.438]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"query \\\"backendControllerConditions\\\" time window 50/289 failed (KQL: let _startTime = datetime(2026-10-15T01:38:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T01:43:14.4931101Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-15T01:38:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T01:43:14.4931101Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n  \"name\": \"backendControllerConditions\"\n}\x1b[0m\n\x1b[37m[21:33:23.712]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgenerating Cosmos content Git history\x1b[0m \x1b[90m{\n  \"input\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/cosmosContent\"\n}\x1b[0m\n\x1b[37m[21:33:23.712]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mParsing log file\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n}\x1b[0m\n\x1b[37m[21:33:24.208]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mFound data dump entries\x1b[0m \x1b[90m{\n  \"count\": 1498,\n  \"files\": 1\n}\x1b[0m\n\x1b[37m[21:33:24.208]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mInitializing git repository\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/cosmosContent\"\n}\x1b[0m\n\x1b[37m[21:33:24.217]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mProcessing entries and creating commits\x1b[0m \x1b[90m{}\x1b[0m\n\x1b[37m[21:33:40.606]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mCompleted\x1b[0m \x1b[90m{\n  \"commits\": 793\n}\x1b[0m\n\x1b[37m[21:33:40.607]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for clusters in the resource group\x1b[0m \x1b[90m{\n  \"clusterIDs\": null,\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect\"\n}\x1b[0m\n\x1b[37m[21:33:40.961]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: cluster IDs to inspect\x1b[0m \x1b[90m{\n  \"clusterIDs\": [\n    \"2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n  ]\n}\x1b[0m\n\x1b[37m[21:33:41.088]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered infra cluster names\x1b[0m \x1b[90m{\n  \"clusterNames\": [\n    \"ci01-j8700672-mgmt-1\",\n    \"ci01-j8700672-svc\"\n  ]\n}\x1b[0m\n\x1b[37m[21:33:41.127]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespaces\": [\n    \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n  ]\n}\x1b[0m\n\x1b[37m[21:33:41.127]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\x1b[0m \x1b[90m{\n  \"managementCluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespaceCount\": 4\n}\x1b[0m\n\x1b[37m[21:33:41.127]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:33:41.317]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:33:41.478]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:33:41.623]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n}\x1b[0m\n\x1b[37m[21:33:43.044]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mmust-gather finished with errors; partial content may have been written\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:21.8572874Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\\\", ActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", SubActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \\u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\\\"}, IsPermanent:false})\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104\"\n}\x1b[0m\n\x1b[37m[21:33:43.044]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mcommand failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:21.8572874Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\\\", ActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", SubActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \\u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\\\"}, IsPermanent:false})\"\n}\x1b[0m\n\n  query-skip-hcp-logs: command failed: exit status 1\noutput: \x1b[37m[21:33:43.407]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843\"\n}\x1b[0m\n\x1b[37m[21:33:49.648]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:47.6716888Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\\\", ActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", SubActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \\u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\\\"}, IsPermanent:false})\",\n  \"name\": \"detailedInfraOrchestrationLogs\"\n}\x1b[0m\n\x1b[37m[21:33:49.648]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"query \\\"backendControllerConditions\\\" time window 47/289 failed (KQL: let _startTime = datetime(2026-10-15T01:23:43.4070857Z);\\nlet _endTime = datetime(2026-10-15T01:28:43.4070856Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n  \"name\": \"backendControllerConditions\"\n}\x1b[0m\n\x1b[37m[21:33:49.744]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgenerating Cosmos content Git history\x1b[0m \x1b[90m{\n  \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/cosmosContent\"\n}\x1b[0m\n\x1b[37m[21:33:49.744]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mParsing log file\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n}\x1b[0m\n\x1b[37m[21:33:50.271]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mFound data dump entries\x1b[0m \x1b[90m{\n  \"count\": 1498,\n  \"files\": 1\n}\x1b[0m\n\x1b[37m[21:33:50.271]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mInitializing git repository\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/cosmosContent\"\n}\x1b[0m\n\x1b[37m[21:33:50.281]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mProcessing entries and creating commits\x1b[0m \x1b[90m{}\x1b[0m\n\x1b[37m[21:34:08.647]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mCompleted\x1b[0m \x1b[90m{\n  \"commits\": 793\n}\x1b[0m\n\x1b[37m[21:34:08.648]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for clusters in the resource group\x1b[0m \x1b[90m{\n  \"clusterIDs\": null,\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect\"\n}\x1b[0m\n\x1b[37m[21:34:09.212]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: cluster IDs to inspect\x1b[0m \x1b[90m{\n  \"clusterIDs\": [\n    \"2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n  ]\n}\x1b[0m\n\x1b[37m[21:34:09.309]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered infra cluster names\x1b[0m \x1b[90m{\n  \"clusterNames\": [\n    \"ci01-j8700672-mgmt-1\",\n    \"ci01-j8700672-svc\"\n  ]\n}\x1b[0m\n\x1b[37m[21:34:09.470]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespaces\": [\n    \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n  ]\n}\x1b[0m\n\x1b[37m[21:34:09.470]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\x1b[0m \x1b[90m{\n  \"managementCluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespaceCount\": 4\n}\x1b[0m\n\x1b[37m[21:34:09.470]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:34:09.615]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:34:10.183]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:34:10.617]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n}\x1b[0m\n\x1b[37m[21:34:12.156]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mmust-gather finished with errors; partial content may have been written\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:47.6716888Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\\\", ActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", SubActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \\u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\\\"}, IsPermanent:false})\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843\"\n}\x1b[0m\n\x1b[37m[21:34:12.157]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mcommand failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:47.6716888Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\\\", ActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", SubActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \\u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\\\"}, IsPermanent:false})\"\n}\x1b[0m\n\n  query-skip-kubernetes-events: command failed: exit status 1\noutput: \x1b[37m[21:34:12.559]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767\"\n}\x1b[0m\n\x1b[37m[21:34:18.220]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:16.3210435Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\\\", ActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", SubActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \\u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\\\"}, IsPermanent:false})\",\n  \"name\": \"detailedInfraOrchestrationLogs\"\n}\x1b[0m\n\x1b[37m[21:34:18.221]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"query \\\"backendControllerConditions\\\" time window 47/289 failed (KQL: let _startTime = datetime(2026-10-15T01:24:12.5582990Z);\\nlet _endTime = datetime(2026-10-15T01:29:12.5582989Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n  \"name\": \"backendControllerConditions\"\n}\x1b[0m\n\x1b[37m[21:34:18.221]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgenerating Cosmos content Git history\x1b[0m \x1b[90m{\n  \"input\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/cosmosContent\"\n}\x1b[0m\n\x1b[37m[21:34:18.221]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mParsing log file\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n}\x1b[0m\n\x1b[37m[21:34:18.723]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mFound data dump entries\x1b[0m \x1b[90m{\n  \"count\": 1498,\n  \"files\": 1\n}\x1b[0m\n\x1b[37m[21:34:18.724]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mInitializing git repository\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/cosmosContent\"\n}\x1b[0m\n\x1b[37m[21:34:18.734]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mProcessing entries and creating commits\x1b[0m \x1b[90m{}\x1b[0m\n\x1b[37m[21:34:35.437]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for clusters in the resource group\x1b[0m \x1b[90m{\n  \"clusterIDs\": null,\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect\"\n}\x1b[0m\n\x1b[37m[21:34:35.845]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: cluster IDs to inspect\x1b[0m \x1b[90m{\n  \"clusterIDs\": [\n    \"2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n  ]\n}\x1b[0m\n\x1b[37m[21:34:36.194]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered infra cluster names\x1b[0m \x1b[90m{\n  \"clusterNames\": [\n    \"ci01-j8700672-mgmt-1\",\n    \"ci01-j8700672-svc\"\n  ]\n}\x1b[0m\n\x1b[37m[21:34:36.313]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespaces\": [\n    \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n  ]\n}\x1b[0m\n\x1b[37m[21:34:36.313]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\x1b[0m \x1b[90m{\n  \"managementCluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespaceCount\": 4\n}\x1b[0m\n\x1b[37m[21:34:36.313]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:34:36.544]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:34:36.863]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:34:37.093]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n}\x1b[0m\n\x1b[37m[21:34:38.438]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mmust-gather finished with errors; partial content may have been written\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:16.3210435Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\\\", ActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", SubActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \\u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767\"\n}\x1b[0m\n\x1b[37m[21:34:38.438]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mcommand failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:16.3210435Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\\\", ActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", SubActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \\u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n}\x1b[0m\n\n  query-collect-systemd-logs: command failed: exit status 1\noutput: \x1b[37m[21:34:38.772]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904\"\n}\x1b[0m\n\x1b[37m[21:34:46.539]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:46.3046137Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\\\", ActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", SubActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \\u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\\\"}, IsPermanent:false})\",\n  \"name\": \"detailedInfraOrchestrationLogs\"\n}\x1b[0m\n\x1b[37m[21:34:46.539]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"query \\\"backendControllerConditions\\\" time window 45/289 failed (KQL: let _startTime = datetime(2026-10-15T01:14:38.7716298Z);\\nlet _endTime = datetime(2026-10-15T01:19:38.7716297Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n  \"name\": \"backendControllerConditions\"\n}\x1b[0m\n\x1b[37m[21:34:47.129]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgenerating Cosmos content Git history\x1b[0m \x1b[90m{\n  \"input\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/cosmosContent\"\n}\x1b[0m\n\x1b[37m[21:34:47.129]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mParsing log file\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n}\x1b[0m\n\x1b[37m[21:34:47.633]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mFound data dump entries\x1b[0m \x1b[90m{\n  \"count\": 1498,\n  \"files\": 1\n}\x1b[0m\n\x1b[37m[21:34:47.633]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mInitializing git repository\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/cosmosContent\"\n}\x1b[0m\n\x1b[37m[21:34:47.642]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mProcessing entries and creating commits\x1b[0m \x1b[90m{}\x1b[0m\n\x1b[37m[21:34:59.291]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for clusters in the resource group\x1b[0m \x1b[90m{\n  \"clusterIDs\": null,\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect\"\n}\x1b[0m\n\x1b[37m[21:34:59.911]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: cluster IDs to inspect\x1b[0m \x1b[90m{\n  \"clusterIDs\": [\n    \"2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n  ]\n}\x1b[0m\n\x1b[37m[21:35:00.221]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered infra cluster names\x1b[0m \x1b[90m{\n  \"clusterNames\": [\n    \"ci01-j8700672-mgmt-1\",\n    \"ci01-j8700672-svc\"\n  ]\n}\x1b[0m\n\x1b[37m[21:35:00.360]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespaces\": [\n    \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n  ]\n}\x1b[0m\n\x1b[37m[21:35:00.360]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\x1b[0m \x1b[90m{\n  \"managementCluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespaceCount\": 4\n}\x1b[0m\n\x1b[37m[21:35:00.360]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:35:00.702]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:35:01.111]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:35:01.638]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n}\x1b[0m\n\x1b[37m[21:35:02.339]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mmust-gather finished with errors; partial content may have been written\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:46.3046137Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\\\", ActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", SubActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \\u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904\"\n}\x1b[0m\n\x1b[37m[21:35:02.340]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mcommand failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:46.3046137Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\\\", ActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", SubActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \\u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n}\x1b[0m\n\n  query-skip-hcp-and-events: command failed: exit status 2\noutput: \x1b[37m[21:35:02.599]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-1950984811\"\n}\x1b[0m\nruntime/cgo: pthread_create failed: Resource temporarily unavailable\nSIGABRT: abort\nPC=0x7f80df60594c m=4 sigcode=18446744073709551610\n\ngoroutine 0 gp=0x14197f1334a0 m=4 mp=0x14197f21a008 [idle]:\nruntime: g 0 gp=0x14197f1334a0: unknown pc 0x7f80df60594c\nstack: frame={sp:0x7f808bffe790, fp:0x0} stack=[0x7f808b7ff1e0,0x7f808bffede0)\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n00007f808bffe690: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6a0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6b0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6c0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6d0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6e0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6f0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe700: 00000000 00000000  00000000 00000000  ................\n00007f808bffe710: 00000000 00000000  00000000 00000000  ................\n00007f808bffe720: 00000000 00000000  00000000 00000000  ................\n00007f808bffe730: 00000000 00000000  00000000 00000000  ................\n00007f808bffe740: 00000000 00000000  00000000 00000000  ................\n00007f808bffe750: 00000000 00000000  00000000 00000000  ................\n00007f808bffe760: 00000000 00000000  00007f80 df612fa9  ........./a.....\n00007f808bffe770: 00000000 00000000  00000000 00000011  ................\n00007f808bffe780: 00000000 00000130  00007f80 df60593e  0.......>Y`.....\n00007f808bffe790: 00000000 00000120  00000000 00000000   ...............\n                  ^ SP\n00007f808bffe7a0: 00010000 00000013  00000000 00000000  ................\n00007f808bffe7b0: 00000000 00000000  00007f80 df688cff  ..........h.....\n00007f808bffe7c0: 00007f80 89ffb910  00007f80 df688cb1  ..........h.....\n00007f808bffe7d0: 00007f80 89ffb640  00007f80 8bffeaa0  @...............\n00007f808bffe7e0: 00007f80 8bffe90e  00007f80 8bffe90f  ................\n00007f808bffe7f0: 00007f80 89ffb640  00007f80 df6037f5  @........7`.....\n00007f808bffe800: 00007f80 84000020  00000000 00000120   ....... .......\n00007f808bffe810: 00000000 003d0f00  701e1ebb c8982900  ..=......).....p\n00007f808bffe820: 00007f80 8bfff640  00000000 00000006  @...............\n00007f808bffe830: 00007f80 84000c90  00007f80 8bffeb60  ........`.......\n00007f808bffe840: 00001419 7f1334a0  00007f80 df5b8646  .4......F.[.....\n00007f808bffe850: 00007f80 df775e90  00007f80 df5a27f3  .^w......'Z.....\n00007f808bffe860: 00000000 00000020  701e1ebb c8982900   ........).....p\n00007f808bffe870: 00007f80 897fb000  00007f80 df677ac4  .........zg.....\n00007f808bffe880: 00000000 00000000  00000000 0000000d  ................\nruntime: g 0 gp=0x14197f1334a0: unknown pc 0x7f80df60594c\nstack: frame={sp:0x7f808bffe790, fp:0x0} stack=[0x7f808b7ff1e0,0x7f808bffede0)\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n00007f808bffe690: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6a0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6b0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6c0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6d0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6e0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6f0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe700: 00000000 00000000  00000000 00000000  ................\n00007f808bffe710: 00000000 00000000  00000000 00000000  ................\n00007f808bffe720: 00000000 00000000  00000000 00000000  ................\n00007f808bffe730: 00000000 00000000  00000000 00000000  ................\n00007f808bffe740: 00000000 00000000  00000000 00000000  ................\n00007f808bffe750: 00000000 00000000  00000000 00000000  ................\n00007f808bffe760: 00000000 00000000  00007f80 df612fa9  ........./a.....\n00007f808bffe770: 00000000 00000000  00000000 00000011  ................\n00007f808bffe780: 00000000 00000130  00007f80 df60593e  0.......>Y`.....\n00007f808bffe790: 00000000 00000120  00000000 00000000   ...............\n                  ^ SP\n00007f808bffe7a0: 00010000 00000013  00000000 00000000  ................\n00007f808bffe7b0: 00000000 00000000  00007f80 df688cff  ..........h.....\n00007f808bffe7c0: 00007f80 89ffb910  00007f80 df688cb1  ..........h.....\n00007f808bffe7d0: 00007f80 89ffb640  00007f80 8bffeaa0  @...............\n00007f808bffe7e0: 00007f80 8bffe90e  00007f80 8bffe90f  ................\n00007f808bffe7f0: 00007f80 89ffb640  00007f80 df6037f5  @........7`.....\n00007f808bffe800: 00007f80 84000020  00000000 00000120   ....... .......\n00007f808bffe810: 00000000 003d0f00  701e1ebb c8982900  ..=......).....p\n00007f808bffe820: 00007f80 8bfff640  00000000 00000006  @...............\n00007f808bffe830: 00007f80 84000c90  00007f80 8bffeb60  ........`.......\n00007f808bffe840: 00001419 7f1334a0  00007f80 df5b8646  .4......F.[.....\n00007f808bffe850: 00007f80 df775e90  00007f80 df5a27f3  .^w......'Z.....\n00007f808bffe860: 00000000 00000020  701e1ebb c8982900   ........).....p\n00007f808bffe870: 00007f80 897fb000  00007f80 df677ac4  .........zg.....\n00007f808bffe880: 00000000 00000000  00000000 0000000d  ................\n\ngoroutine 1 gp=0x14197f1321e0 m=nil [sync.WaitGroup.Wait]:\nruntime.gopark(0x14197fe86c00?, 0x2022a4d?, 0x10?, 0x63?, 0x14197fe86c30?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197ff32be8 sp=0x14197ff32bc8 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x14197fc98610, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197ff32c50 sp=0x14197ff32be8 pc=0x1f162b2\nsync.runtime_SemacquireWaitGroup(0x14197fd289c0?, 0xe0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x14197ff32c88 sp=0x14197ff32c50 pc=0x1f3a66e\nsync.(*WaitGroup).Wait(0x14197fc98608)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x14197ff32cb0 sp=0x14197ff32c88 pc=0x1f4f445\ngolang.org/x/sync/errgroup.(*Group).Wait(0x14197fc98600)\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x14197ff32cd0 sp=0x14197ff32cb0 pc=0x362507e\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x14197f521dc0, {0x59f0b08, 0x14197f530fc0}, {0x14197fcca000, 0x15, 0x1c})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x14197ff32d90 sp=0x14197ff32cd0 pc=0x3927e05\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x14197f521dc0, {0x59f0b08, 0x14197f530fc0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:500 +0x4b08 fp=0x14197ff33968 sp=0x14197ff32d90 pc=0x39261c8\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedQueryOptions).RunQuery(0x14197f1401e0, {0x59f0b08, 0x14197f530fc0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_cmd.go:75 +0x565 fp=0x14197ff33b30 sp=0x14197ff33968 pc=0x3937005\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawQueryOptions).Run(0x0?, {0x59f0b08, 0x14197f530fc0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_options.go:178 +0x54 fp=0x14197ff33b58 sp=0x14197ff33b30 pc=0x3939d34\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryCommand.func1(0x14197f4ba008?, {0x14197f5f82a0?, 0x4?, 0x553b0e2?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_cmd.go:45 +0x28 fp=0x14197ff33b80 sp=0x14197ff33b58 pc=0x3936a68\ngithub.com/spf13/cobra.(*Command).execute(0x14197f4ba008, {0x14197f2c0030, 0xe, 0xf})\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x14197ff33d28 sp=0x14197ff33b80 pc=0x212e614\ngithub.com/spf13/cobra.(*Command).ExecuteC(0x14197f625b08)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x14197ff33e18 sp=0x14197ff33d28 pc=0x212efa5\ngithub.com/spf13/cobra.(*Command).Execute(...)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071\ngithub.com/spf13/cobra.(*Command).ExecuteContext(...)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064\nmain.main()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x14197ff33f48 sp=0x14197ff33e18 pc=0x47ddd6a\nruntime.main()\n\t/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x14197ff33fe0 sp=0x14197ff33f48 pc=0x1f01975\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197ff33fe8 sp=0x14197ff33fe0 pc=0x1f40801\n\ngoroutine 2 gp=0x14197f132d20 m=nil [force gc (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f2fa8 sp=0x14197f1f2f88 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.forcegchelper()\n\t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x14197f1f2fe0 sp=0x14197f1f2fa8 pc=0x1f01c93\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f2fe8 sp=0x14197f1f2fe0 pc=0x1f40801\ncreated by runtime.init.7 in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\n\ngoroutine 18 gp=0x14197f2843c0 m=nil [GC sweep wait]:\nruntime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1ee788 sp=0x14197f1ee768 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.bgsweep(0x14197f292000)\n\t/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x14197f1ee7c8 sp=0x14197f1ee788 pc=0x1ee9f11\nruntime.gcenable.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x14197f1ee7e0 sp=0x14197f1ee7c8 pc=0x1edadd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1ee7e8 sp=0x14197f1ee7e0 pc=0x1f40801\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\n\ngoroutine 19 gp=0x14197f2845a0 m=nil [GC scavenge wait]:\nruntime.gopark(0x138d6d?, 0x3979fd?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1eef78 sp=0x14197f1eef58 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.(*scavengerState).park(0x82979c0)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x14197f1eefa8 sp=0x14197f1eef78 pc=0x1ee77e9\nruntime.bgscavenge(0x14197f292000)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x14197f1eefc8 sp=0x14197f1eefa8 pc=0x1ee7d99\nruntime.gcenable.gowrap2()\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x14197f1eefe0 sp=0x14197f1eefc8 pc=0x1edad97\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1eefe8 sp=0x14197f1eefe0 pc=0x1f40801\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\n\ngoroutine 20 gp=0x14197f284780 m=nil [GOMAXPROCS updater (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1ef788 sp=0x14197f1ef768 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.updateMaxProcsGoroutine()\n\t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x14197f1ef7e0 sp=0x14197f1ef788 pc=0x1f10267\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1ef7e8 sp=0x14197f1ef7e0 pc=0x1f40801\ncreated by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\n\ngoroutine 21 gp=0x14197f284d20 m=nil [finalizer wait]:\nruntime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x14197f1f2601?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f2620 sp=0x14197f1f2600 pc=0x1f38d2e\nruntime.runFinalizers()\n\t/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x14197f1f27e0 sp=0x14197f1f2620 pc=0x1ed9cc7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f27e8 sp=0x14197f1f27e0 pc=0x1f40801\ncreated by runtime.createfing in goroutine 1\n\t/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d\n\ngoroutine 22 gp=0x14197f47c960 m=nil [cleanup wait]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1eff68 sp=0x14197f1eff48 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.(*cleanupQueue).dequeue(0x82980e0)\n\t/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x14197f1effa0 sp=0x14197f1eff68 pc=0x1ed6074\nruntime.runCleanups()\n\t/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x14197f1effe0 sp=0x14197f1effa0 pc=0x1ed66e5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1effe8 sp=0x14197f1effe0 pc=0x1f40801\ncreated by runtime.(*cleanupQueue).createGs in goroutine 1\n\t/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5\n\ngoroutine 23 gp=0x14197f5a30e0 m=nil [GC worker (idle)]:\nruntime.gopark(0x125b862b4ec7?, 0x1?, 0x21?, 0xab?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f0740 sp=0x14197f1f0720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x14197f29d9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f07c8 sp=0x14197f1f0740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f07e0 sp=0x14197f1f07c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f07e8 sp=0x14197f1f07e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 3 gp=0x14197f133680 m=nil [runnable]:\nruntime.gopark(0x125b8625d223?, 0x3?, 0x89?, 0x19?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f3740 sp=0x14197f1f3720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x14197f29d9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f37c8 sp=0x14197f1f3740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f37e0 sp=0x14197f1f37c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f37e8 sp=0x14197f1f37e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 34 gp=0x14197f604000 m=nil [GC worker (idle)]:\nruntime.gopark(0x125b862e8e02?, 0x3?, 0xe6?, 0x2d?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60a740 sp=0x14197f60a720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x14197f29d9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f60a7c8 sp=0x14197f60a740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f60a7e0 sp=0x14197f60a7c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60a7e8 sp=0x14197f60a7e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 24 gp=0x14197f5a32c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x125b862118da?, 0x1?, 0x4d?, 0x9a?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f0f40 sp=0x14197f1f0f20 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x14197f29d9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f0fc8 sp=0x14197f1f0f40 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f0fe0 sp=0x14197f1f0fc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f0fe8 sp=0x14197f1f0fe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 4 gp=0x14197f133860 m=nil [GC worker (idle)]:\nruntime.gopark(0x125b7fe90140?, 0x3?, 0x20?, 0x3a?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f3f40 sp=0x14197f1f3f20 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x14197f29d9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f3fc8 sp=0x14197f1f3f40 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f3fe0 sp=0x14197f1f3fc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f3fe8 sp=0x14197f1f3fe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 35 gp=0x14197f6041e0 m=nil [GC worker (idle)]:\nruntime.gopark(0x125b862a34bd?, 0x3?, 0x4?, 0x15?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60af40 sp=0x14197f60af20 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x14197f29d9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f60afc8 sp=0x14197f60af40 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f60afe0 sp=0x14197f60afc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60afe8 sp=0x14197f60afe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 25 gp=0x14197f5a34a0 m=3 mp=0x14197f1a5808 [GC mark termination]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x14197f1f16a0 sp=0x14197f1f1690 pc=0x1f3ebc8\nruntime.forEachP(...)\n\t/usr/lib/golang/src/runtime/proc.go:2112\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x14197f1f1740 sp=0x14197f1f16a0 pc=0x1edbc34\nruntime.gcBgMarkWorker(0x14197f29d9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x14197f1f17c8 sp=0x14197f1f1740 pc=0x1edda88\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f17e0 sp=0x14197f1f17c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f17e8 sp=0x14197f1f17e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 5 gp=0x14197f133a40 m=nil [GC worker (idle)]:\nruntime.gopark(0x125b8624ea9b?, 0x1?, 0xfd?, 0x7a?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f4740 sp=0x14197f1f4720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x14197f29d9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f47c8 sp=0x14197f1f4740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f47e0 sp=0x14197f1f47c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f47e8 sp=0x14197f1f47e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 50 gp=0x14197f768f00 m=nil [select, locked to thread]:\nruntime.gopark(0x14197f60c7a8?, 0x2?, 0x1?, 0x8f?, 0x14197f60c794?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60c628 sp=0x14197f60c608 pc=0x1f38d2e\nruntime.selectgo(0x14197f60c7a8, 0x14197f60c790, 0x0?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f60c758 sp=0x14197f60c628 pc=0x1f152a5\nruntime.ensureSigM.func1()\n\t/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x14197f60c7e0 sp=0x14197f60c758 pc=0x1f328e8\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60c7e8 sp=0x14197f60c7e0 pc=0x1f40801\ncreated by runtime.ensureSigM in goroutine 1\n\t/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5\n\ngoroutine 26 gp=0x14197f3b34a0 m=7 mp=0x14197f55b008 [syscall]:\nruntime.notetsleepg(0x82bdea0, 0xffffffffffffffff)\n\t/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x14197f6077a0 sp=0x14197f607778 pc=0x1ece509\nos/signal.signal_recv()\n\t/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x14197f6077c0 sp=0x14197f6077a0 pc=0x1f3b298\nos/signal.loop()\n\t/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x14197f6077e0 sp=0x14197f6077c0 pc=0x2030553\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f6077e8 sp=0x14197f6077e0 pc=0x1f40801\ncreated by os/signal.Notify.func1.1 in goroutine 1\n\t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\n\ngoroutine 27 gp=0x14197f6894a0 m=nil [select]:\nruntime.gopark(0x14197f607f98?, 0x2?, 0x68?, 0x81?, 0x14197f607f8c?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f607e28 sp=0x14197f607e08 pc=0x1f38d2e\nruntime.selectgo(0x14197f607f98, 0x14197f607f88, 0x0?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f607f58 sp=0x14197f607e28 pc=0x1f152a5\nos/signal.NotifyContext.func1()\n\t/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x14197f607fe0 sp=0x14197f607f58 pc=0x202ff9c\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f607fe8 sp=0x14197f607fe0 pc=0x1f40801\ncreated by os/signal.NotifyContext in goroutine 1\n\t/usr/lib/golang/src/os/signal/signal.go:291 +0x158\n\ngoroutine 224 gp=0x14197f689680 m=nil [semacquire]:\nruntime.gopark(0x7f80904adba8?, 0x1eed7e0?, 0xc0?, 0x68?, 0x4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419800b5b58 sp=0x1419800b5b38 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x1419800b5bc0 sp=0x1419800b5b58 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x1419800b5c60 sp=0x1419800b5bc0 pc=0x1edbb12\nruntime.gcAssistAlloc(0x14197f689680)\n\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x1419800b5cf0 sp=0x1419800b5c60 pc=0x1ee06b6\nruntime.deductAssistCredit(0x7ffce51f9080?)\n\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x1419800b5d10 sp=0x1419800b5cf0 pc=0x1ed10a6\nruntime.mallocgc(0x7fe00, 0x4af32a0, 0x1)\n\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x1419800b5d40 sp=0x1419800b5d10 pc=0x1f35e07\nruntime.makeslice(0x14197fc9d0b0?, 0x141983a1fe00?, 0x20000?)\n\t/usr/lib/golang/src/runtime/slice.go:117 +0x49 fp=0x1419800b5d68 sp=0x1419800b5d40 pc=0x1f3b489\nencoding/json.(*Decoder).refill(0x14197f658c80)\n\t/usr/lib/golang/src/encoding/json/stream.go:161 +0xee fp=0x1419800b5db8 sp=0x1419800b5d68 pc=0x200d48e\nencoding/json.(*Decoder).readValue(0x14197f658c80)\n\t/usr/lib/golang/src/encoding/json/stream.go:142 +0x85 fp=0x1419800b5e08 sp=0x1419800b5db8 pc=0x200d125\nencoding/json.(*Decoder).Decode(0x14197f658c80, {0x4cfb340, 0x14197ffa44c0})\n\t/usr/lib/golang/src/encoding/json/stream.go:65 +0x75 fp=0x1419800b5e38 sp=0x1419800b5e08 pc=0x200cdf5\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readPrimaryTable(0x14197f8c54f0, 0x14197fb50000)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:264 +0x16e fp=0x1419800b5ec8 sp=0x1419800b5e38 pc=0x38d822e\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x14197f8c54f0)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:174 +0x189 fp=0x1419800b5f30 sp=0x1419800b5ec8 pc=0x38d7b89\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x14197f8c54f0, 0x14198007be90)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x1419800b5fc0 sp=0x1419800b5f30 pc=0x38d788c\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x1419800b5fe0 sp=0x1419800b5fc0 pc=0x38d751b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1419800b5fe8 sp=0x1419800b5fe0 pc=0x1f40801\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 183\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc\n\ngoroutine 87 gp=0x14197fd3c000 m=nil [sync.WaitGroup.Wait]:\nruntime.gopark(0x14197fcf9e00?, 0x1ed0734?, 0x20?, 0x94?, 0x7f80917bdf06?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf9da0 sp=0x14197fcf9d80 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x14197f69cc50, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fcf9e08 sp=0x14197fcf9da0 pc=0x1f162b2\nsync.runtime_SemacquireWaitGroup(0x14197f86b2d8?, 0x0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x14197fcf9e40 sp=0x14197fcf9e08 pc=0x1f3a66e\nsync.(*WaitGroup).Wait(0x14197f69cc48)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x14197fcf9e68 sp=0x14197fcf9e40 pc=0x1f4f445\ngolang.org/x/sync/errgroup.(*Group).Wait(0x14197f69cc40)\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x14197fcf9e88 sp=0x14197fcf9e68 pc=0x362507e\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x14197f521d80, {0x59f0b40, 0x14197f8c4b40}, {0x14197fcca000, 0x15, 0x6f63746669687372?}, 0x14197f1c02a0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x14197fcf9f00 sp=0x14197fcf9e88 pc=0x392976a\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x14197fcf9f78 sp=0x14197fcf9f00 pc=0x392808e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197fcf9fe0 sp=0x14197fcf9f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf9fe8 sp=0x14197fcf9fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 77 gp=0x14197fd3c1e0 m=nil [IO wait]:\nruntime.gopark(0x14197fb3e4a0?, 0x1edb176?, 0x80?, 0xed?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f202808 sp=0x14197f2027e8 pc=0x1f38d2e\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x14197f202840 sp=0x14197f202808 pc=0x1ef9e57\ninternal/poll.runtime_pollWait(0x7f8098109000, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x14197f202860 sp=0x14197f202840 pc=0x1f375c5\ninternal/poll.(*pollDesc).wait(0x14197f323a80?, 0x14197f34d300?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x14197f202888 sp=0x14197f202860 pc=0x1fbd167\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0x14197f323a80, {0x14197f34d300, 0x1300, 0x1300})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x14197f202920 sp=0x14197f202888 pc=0x1fbe4ce\nnet.(*netFD).Read(0x14197f323a80, {0x14197f34d300?, 0x14197f34d30a?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x14197f202968 sp=0x14197f202920 pc=0x20798c5\nnet.(*conn).Read(0x14197f1300d8, {0x14197f34d300?, 0x7f809057a868?, 0x7f80df53a6d8?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x14197f2029b0 sp=0x14197f202968 pc=0x208b7a5\ncrypto/tls.(*atLeastReader).Read(0x14197f821a10, {0x14197f34d300?, 0x14197f202a98?, 0x1efcc28?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x14197f2029f8 sp=0x14197f2029b0 pc=0x2222ddb\nbytes.(*Buffer).ReadFrom(0x14197f8a57a8, {0x59a3680, 0x14197f821a10})\n\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x14197f202a50 sp=0x14197f2029f8 pc=0x1fe9258\ncrypto/tls.(*Conn).readFromUntil(0x14197f8a5508, {0x59a32c0, 0x14197f1300d8}, 0x222764c?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x14197f202a88 sp=0x14197f202a50 pc=0x2222fbe\ncrypto/tls.(*Conn).readRecordOrCCS(0x14197f8a5508, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x14197f202c68 sp=0x14197f202a88 pc=0x222009b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:588\ncrypto/tls.(*Conn).Read(0x14197f8a5508, {0x14197fa26000, 0x1000, 0x4e55540?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x14197f202cd0 sp=0x14197f202c68 pc=0x22267c5\nnet/http.(*persistConn).Read(0x14197f8032c0, {0x14197fa26000?, 0x59a0ea0?, 0x81260e0?})\n\t/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x14197f202d30 sp=0x14197f202cd0 pc=0x2319d07\nbufio.(*Reader).fill(0x14197f29b620)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f202d68 sp=0x14197f202d30 pc=0x20a6c63\nbufio.(*Reader).Peek(0x14197f29b620, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x14197f202d80 sp=0x14197f202d68 pc=0x20a6df3\nnet/http.(*persistConn).readLoop(0x14197f8032c0)\n\t/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x14197f202fc8 sp=0x14197f202d80 pc=0x231a812\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x14197f202fe0 sp=0x14197f202fc8 pc=0x2319197\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f202fe8 sp=0x14197f202fe0 pc=0x1f40801\ncreated by net/http.(*Transport).dialConn in goroutine 9\n\t/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac\n\ngoroutine 7 gp=0x14197fe025a0 m=nil [sync.Mutex.Lock]:\nruntime.gopark(0x14197f9d3c68?, 0x14197f9d3c70?, 0x94?, 0x69?, 0x14197f9d3c78?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f9d3c00 sp=0x14197f9d3be0 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x14197fccaad4, 0x0, 0x3, 0x2, 0x16)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197f9d3c68 sp=0x14197f9d3c00 pc=0x1f162b2\ninternal/sync.runtime_SemacquireMutex(0x1000?, 0xd0?, 0x1ed11a5?)\n\t/usr/lib/golang/src/runtime/sema.go:95 +0x25 fp=0x14197f9d3ca0 sp=0x14197f9d3c68 pc=0x1f3a545\ninternal/sync.(*Mutex).lockSlow(0x14197fccaad0)\n\t/usr/lib/golang/src/internal/sync/mutex.go:149 +0x15d fp=0x14197f9d3cf0 sp=0x14197f9d3ca0 pc=0x1f4d31d\ninternal/sync.(*Mutex).Lock(...)\n\t/usr/lib/golang/src/internal/sync/mutex.go:70\nsync.(*Mutex).Lock(...)\n\t/usr/lib/golang/src/sync/mutex.go:46\nnet/http.(*http2clientConnReadLoop).streamByID(0x22babb5?, 0x1b, 0x1)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:10083 +0x5b fp=0x14197f9d3d48 sp=0x14197f9d3cf0 pc=0x22e10db\nnet/http.(*http2clientConnReadLoop).processData(0x14197f9d3fa8, 0x14197fc9d200)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9939 +0x45 fp=0x14197f9d3e18 sp=0x14197f9d3d48 pc=0x22e05e5\nnet/http.(*http2clientConnReadLoop).run(0x14197f9d3fa8)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9584 +0x4b6 fp=0x14197f9d3f60 sp=0x14197f9d3e18 pc=0x22de536\nnet/http.(*http2ClientConn).readLoop(0x14197fccaa80)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x14197f9d3fc8 sp=0x14197f9d3f60 pc=0x22dd532\nnet/http.(*http2Transport).newClientConn.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x14197f9d3fe0 sp=0x14197f9d3fc8 pc=0x22d7057\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f9d3fe8 sp=0x14197f9d3fe0 pc=0x1f40801\ncreated by net/http.(*http2Transport).newClientConn in goroutine 6\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5\n\ngoroutine 78 gp=0x14197fe02960 m=nil [select]:\nruntime.gopark(0x14197f9f7f48?, 0x2?, 0x0?, 0xb6?, 0x14197f9f7ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf2d88 sp=0x14197fcf2d68 pc=0x1f38d2e\nruntime.selectgo(0x14197fcf2f48, 0x14197f9f7ef0, 0x14197f1cd0c0?, 0x0, 0x14197f89c2a0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197fcf2eb8 sp=0x14197fcf2d88 pc=0x1f152a5\nnet/http.(*persistConn).writeLoop(0x14197f8032c0)\n\t/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x14197fcf2fc8 sp=0x14197fcf2eb8 pc=0x231c426\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x14197fcf2fe0 sp=0x14197fcf2fc8 pc=0x2319157\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf2fe8 sp=0x14197fcf2fe0 pc=0x1f40801\ncreated by net/http.(*Transport).dialConn in goroutine 9\n\t/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe\n\ngoroutine 163 gp=0x14197f7694a0 m=nil [select]:\nruntime.gopark(0x14197fdd3f08?, 0x6?, 0xa0?, 0x94?, 0x14197fdd3da4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f205c08 sp=0x14197f205be8 pc=0x1f38d2e\nruntime.selectgo(0x14197f205f08, 0x14197fdd3d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f205d38 sp=0x14197f205c08 pc=0x1f152a5\nnet/http.(*http2clientStream).writeRequest(0x14197f37f380, 0x14197f49e140, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x14197f205f90 sp=0x14197f205d38 pc=0x22da6a8\nnet/http.(*http2clientStream).doRequest(0x14197f37f380, 0x14197f546100?, 0x14197f609798?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x14197f205fb8 sp=0x14197f205f90 pc=0x22d99d8\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197f205fe0 sp=0x14197f205fb8 pc=0x22d999f\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f205fe8 sp=0x14197f205fe0 pc=0x1f40801\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 184\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n\ngoroutine 88 gp=0x14197f769a40 m=nil [semacquire]:\nruntime.gopark(0x141980119958?, 0x1ed3e05?, 0xe8?, 0xd2?, 0x7f80904ad2e8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x141980119908 sp=0x1419801198e8 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x141980119970 sp=0x141980119908 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x141980119a10 sp=0x141980119970 pc=0x1edbb12\nruntime.gcAssistAlloc(0x14197f769a40)\n\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x141980119aa0 sp=0x141980119a10 pc=0x1ee06b6\nruntime.deductAssistCredit(0x141983b00000?)\n\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x141980119ac0 sp=0x141980119aa0 pc=0x1ed10a6\nruntime.mallocgc(0x7edd, 0x0, 0x0)\n\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x141980119af0 sp=0x141980119ac0 pc=0x1f35e07\nruntime.slicebytetostring(0x4e537a0?, 0x141983b00000, 0x7edd)\n\t/usr/lib/golang/src/runtime/string.go:170 +0x77 fp=0x141980119b20 sp=0x141980119af0 pc=0x1f1e397\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x14197f8c4b40}, 0x14197f1c03f0, 0x14197f531740)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:322 +0x91b fp=0x141980119f48 sp=0x141980119b20 pc=0x3920bfb\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x141980119f78 sp=0x141980119f48 pc=0x392800c\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x141980119fe0 sp=0x141980119f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x141980119fe8 sp=0x141980119fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 110 gp=0x14197fd3cb40 m=nil [sync.Mutex.Lock]:\nruntime.gopark(0x22b8a73?, 0x14197f7e1000?, 0xb0?, 0x4c?, 0x8160a40?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1c7510 sp=0x14197f1c74f0 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x14197fccaad4, 0x0, 0x3, 0x2, 0x16)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197f1c7578 sp=0x14197f1c7510 pc=0x1f162b2\ninternal/sync.runtime_SemacquireMutex(0x14197f1c75d0?, 0x66?, 0x61f?)\n\t/usr/lib/golang/src/runtime/sema.go:95 +0x25 fp=0x14197f1c75b0 sp=0x14197f1c7578 pc=0x1f3a545\ninternal/sync.(*Mutex).lockSlow(0x14197fccaad0)\n\t/usr/lib/golang/src/internal/sync/mutex.go:149 +0x15d fp=0x14197f1c7600 sp=0x14197f1c75b0 pc=0x1f4d31d\ninternal/sync.(*Mutex).Lock(...)\n\t/usr/lib/golang/src/internal/sync/mutex.go:70\nsync.(*Mutex).Lock(...)\n\t/usr/lib/golang/src/sync/mutex.go:46\nnet/http.http2transportResponseBody.Read({0x1f3ec12?}, {0x14198313d000?, 0x1eed797?, 0x14197f1c76b0?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9875 +0x209 fp=0x14197f1c7688 sp=0x14197f1c7600 pc=0x22dff89\nbufio.(*Reader).fill(0x14197f3a00c0)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f1c76c0 sp=0x14197f1c7688 pc=0x20a6c63\nbufio.(*Reader).ReadByte(0x14197f3a00c0)\n\t/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x14197f1c76d8 sp=0x14197f1c76c0 pc=0x20a73c7\ncompress/flate.(*decompressor).moreBits(0x141980059908)\n\t/usr/lib/golang/src/compress/flate/inflate.go:697 +0x24 fp=0x14197f1c7710 sp=0x14197f1c76d8 pc=0x2288824\ncompress/flate.(*decompressor).nextBlock(0x141980059908)\n\t/usr/lib/golang/src/compress/flate/inflate.go:304 +0x28 fp=0x14197f1c7728 sp=0x14197f1c7710 pc=0x2286ba8\ncompress/flate.(*decompressor).Read(0x141980059908, {0x14198314e000, 0x1000, 0x7f8090ef0f00?})\n\t/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x14197f1c7758 sp=0x14197f1c7728 pc=0x2286dbb\ncompress/gzip.(*Reader).Read(0x14197f672dc8, {0x14198314e000, 0x1000, 0x1000})\n\t/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x14197f1c77e8 sp=0x14197f1c7758 pc=0x228c838\ngithub.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x410d0fa76c5d7d40?, {0x14198314e000?, 0x7f8090ef0d20?, 0x52ffb40?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x14197f1c7818 sp=0x14197f1c77e8 pc=0x38b33e2\nbufio.(*Reader).fill(0x14197f3a0120)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f1c7850 sp=0x14197f1c7818 pc=0x20a6c63\nbufio.(*Reader).Peek(0x14197f3a0120, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x14197f1c7868 sp=0x14197f1c7850 pc=0x20a6df3\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.validateJsonResponse(0x14197f3a0120)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:38 +0x25 fp=0x14197f1c78b8 sp=0x14197f1c7868 pc=0x38d6be5\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.newFrameReader({0x59db110, 0x14197fe1e040}, {0x59f0b40, 0x14197f748140})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:26 +0x188 fp=0x14197f1c7960 sp=0x14197f1c78b8 pc=0x38d6a08\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset({0x59f0bb0?, 0x14197f4797a0?}, {0x59db110, 0x14197fe1e040}, 0x1, 0x3e8, 0x1)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:66 +0x19e fp=0x14197f1c79e0 sp=0x14197f1c7960 pc=0x38d737e\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x14197f521d40, {0x59f0bb0, 0x14197f4797a0}, {0x14197f49a330, 0xb}, 0x14197fe1e700, {0x0?, 0x2?, 0x0?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:167 +0x2b7 fp=0x14197f1c7a78 sp=0x14197f1c79e0 pc=0x38e4f17\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197f7a2600}, 0x14197f1c02a0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x14197f1c7d60 sp=0x14197f1c7a78 pc=0x391b5c5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x14197f521d80, {0x59f0b40, 0x14197fdd8140}, {0x5a00340, 0x14197fc1e5d0}, 0x14197f1c02a0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:98 +0x366 fp=0x14197f1c7e88 sp=0x14197f1c7d60 pc=0x3929ec6\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14197f1c7f78 sp=0x14197f1c7e88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197f1c7fe0 sp=0x14197f1c7f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1c7fe8 sp=0x14197f1c7fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 89 gp=0x14197fd82960 m=nil [select]:\nruntime.gopark(0x1419800b7d00?, 0x2?, 0xd7?, 0xfd?, 0x1419800b7c74?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419800b7ac8 sp=0x1419800b7aa8 pc=0x1f38d2e\nruntime.selectgo(0x1419800b7d00, 0x1419800b7c70, 0x1419801521c0?, 0x0, 0x14197f5daba0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x1419800b7bf8 sp=0x1419800b7ac8 pc=0x1f152a5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x14197f521d80?, {0x59f0b40, 0x14197f8c4b40}, 0x14197f1c02a0, 0x14197f1c03f0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x1419800b7f10 sp=0x1419800b7bf8 pc=0x392836e\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x1419800b7f78 sp=0x1419800b7f10 pc=0x3927f3d\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x1419800b7fe0 sp=0x1419800b7f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1419800b7fe8 sp=0x1419800b7fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 175 gp=0x14197fd3d2c0 m=nil [sync.Cond.Wait]:\nruntime.gopark(0x22c2d5b?, 0x14197fe1c080?, 0x0?, 0xe0?, 0x14197f777000?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14198012b978 sp=0x14198012b958 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nsync.runtime_notifyListWait(0x14197f37f3c8, 0x3c)\n\t/usr/lib/golang/src/runtime/sema.go:617 +0x1b3 fp=0x14198012b9c8 sp=0x14198012b978 pc=0x1f3a8f3\nsync.(*Cond).Wait(0x1000?)\n\t/usr/lib/golang/src/sync/cond.go:71 +0x73 fp=0x14198012ba00 sp=0x14198012b9c8 pc=0x1f4d673\nnet/http.(*http2pipe).Read(0x14197f37f3b0, {0x14197ff7e000, 0x1000, 0x1000})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:3928 +0xdb fp=0x14198012ba68 sp=0x14198012ba00 pc=0x22c2c1b\nnet/http.http2transportResponseBody.Read({0x4a54?}, {0x14197ff7e000?, 0x14198012bb80?, 0x2287949?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9852 +0x59 fp=0x14198012baf0 sp=0x14198012ba68 pc=0x22dfdd9\nbufio.(*Reader).fill(0x14197fcaa000)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14198012bb28 sp=0x14198012baf0 pc=0x20a6c63\nbufio.(*Reader).ReadByte(0x14197fcaa000)\n\t/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x14198012bb40 sp=0x14198012bb28 pc=0x20a73c7\ncompress/flate.(*decompressor).moreBits(0x141980056008)\n\t/usr/lib/golang/src/compress/flate/inflate.go:697 +0x24 fp=0x14198012bb78 sp=0x14198012bb40 pc=0x2288824\ncompress/flate.(*decompressor).nextBlock(0x141980056008)\n\t/usr/lib/golang/src/compress/flate/inflate.go:304 +0x28 fp=0x14198012bb90 sp=0x14198012bb78 pc=0x2286ba8\ncompress/flate.(*decompressor).Read(0x141980056008, {0x141980034b13, 0x4ed, 0xb13?})\n\t/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x14198012bbc0 sp=0x14198012bb90 pc=0x2286dbb\ncompress/gzip.(*Reader).Read(0x141980000b08, {0x141980034b13, 0x4ed, 0x4ed})\n\t/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x14198012bc50 sp=0x14198012bbc0 pc=0x228c838\ngithub.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x14198012bcf0?, {0x141980034b13?, 0x1000?, 0x0?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x14198012bc80 sp=0x14198012bc50 pc=0x38b33e2\nbufio.(*Reader).fill(0x141980807010)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14198012bcb8 sp=0x14198012bc80 pc=0x20a6c63\nbufio.(*Reader).ReadSlice(0x141980807010, 0xa)\n\t/usr/lib/golang/src/bufio/bufio.go:377 +0x29 fp=0x14198012bd00 sp=0x14198012bcb8 pc=0x20a7769\nbufio.(*Reader).collectFragments(0x141980807010, 0xa)\n\t/usr/lib/golang/src/bufio/bufio.go:452 +0x85 fp=0x14198012bde0 sp=0x14198012bd00 pc=0x20a7b85\nbufio.(*Reader).ReadBytes(0x4bbc5a0?, 0xa0?)\n\t/usr/lib/golang/src/bufio/bufio.go:479 +0x17 fp=0x14198012be60 sp=0x14198012bde0 pc=0x20a7e77\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.(*frameReader).advance(0x141980807000)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:64 +0x45 fp=0x14198012beb0 sp=0x14198012be60 pc=0x38d6d45\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readRoutine(0x141980807000, 0x14197f8c8370)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:85 +0x51 fp=0x14198012bfc0 sp=0x14198012beb0 pc=0x38d7591\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap2()\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x1b fp=0x14198012bfe0 sp=0x14198012bfc0 pc=0x38d74db\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14198012bfe8 sp=0x14198012bfe0 pc=0x1f40801\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 184\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x24a\n\ngoroutine 183 gp=0x14197fd3d860 m=nil [semacquire]:\nruntime.gopark(0x7f8090f36c08?, 0x1eed7e0?, 0x80?, 0x43?, 0x2?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fac5208 sp=0x14197fac51e8 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fac5270 sp=0x14197fac5208 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x14197fac5310 sp=0x14197fac5270 pc=0x1edbb12\nruntime.gcAssistAlloc(0x14197fd3d860)\n\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x14197fac53a0 sp=0x14197fac5310 pc=0x1ee06b6\nruntime.deductAssistCredit(0x14197fac53f0?)\n\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x14197fac53c0 sp=0x14197fac53a0 pc=0x1ed10a6\nruntime.mallocgc(0x5500, 0x0, 0x0)\n\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x14197fac53f0 sp=0x14197fac53c0 pc=0x1f35e07\nruntime.growslice(0x141983adc000, 0x14197fac54a8?, 0x1ed3be5?, 0x82bed80?, 0x1ed0417?)\n\t/usr/lib/golang/src/runtime/slice.go:265 +0x573 fp=0x14197fac5460 sp=0x14197fac53f0 pc=0x1f3ba93\nstrings.(*Builder).Write(0x14197f569ea0, {0x1419839fd000, 0x1000, 0x1000000010765?})\n\t/usr/lib/golang/src/strings/builder.go:89 +0x90 fp=0x14197fac54b0 sp=0x14197fac5460 pc=0x1ff0cb0\nbufio.(*Writer).Flush(0x14197fac5698)\n\t/usr/lib/golang/src/bufio/bufio.go:642 +0x52 fp=0x14197fac54f8 sp=0x14197fac54b0 pc=0x20a8832\nbufio.(*Writer).WriteString(0x14197fac5698, {0x553a4f6?, 0x553a9e6?})\n\t/usr/lib/golang/src/bufio/bufio.go:766 +0x165 fp=0x14197fac5568 sp=0x14197fac54f8 pc=0x20a8f65\nencoding/csv.(*Writer).Write(0x14197fac5688, {0x14198384d480?, 0x4, 0x141983909500?})\n\t/usr/lib/golang/src/encoding/csv/writer.go:92 +0x2b4 fp=0x14197fac55d0 sp=0x14197fac5568 pc=0x20ad3d4\ngithub.com/Azure/azure-kusto-go/azkustodata/query.(*row).String(0x5e?)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/row_impl.go:101 +0x21f fp=0x14197fac56e8 sp=0x14197fac55d0 pc=0x38cf67f\nfmt.(*pp).handleMethods(0x14197fc20000, 0x7fac58f0?)\n\t/usr/lib/golang/src/fmt/print.go:674 +0x2f4 fp=0x14197fac58a8 sp=0x14197fac56e8 pc=0x1fdd1d4\nfmt.(*pp).printArg(0x14197fc20000, {0x549a3a0, 0x14197fd92e80}, 0x76)\n\t/usr/lib/golang/src/fmt/print.go:757 +0x42e fp=0x14197fac5938 sp=0x14197fac58a8 pc=0x1fdd9ee\nfmt.(*pp).doPrintf(0x14197fc20000, {0x553a400, 0x2}, {0x14197fac5d08, 0x1, 0x1})\n\t/usr/lib/golang/src/fmt/print.go:1075 +0x3ec fp=0x14197fac5a20 sp=0x14197fac5938 pc=0x1fe04ec\nfmt.Sprintf({0x553a400, 0x2}, {0x14197fac5d08, 0x1, 0x1})\n\t/usr/lib/golang/src/fmt/print.go:239 +0x53 fp=0x14197fac5a78 sp=0x14197fac5a20 pc=0x1fda6b3\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197fc4b200}, 0x14197f1c02a0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:160 +0x8d8 fp=0x14197fac5d60 sp=0x14197fac5a78 pc=0x391bad8\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x14197fdd8140?}, {0x5a00300?, 0x14197fc4b200?}, 0x0?)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x14197fac5e88 sp=0x14197fac5d60 pc=0x3929c16\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14197fac5f78 sp=0x14197fac5e88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197fac5fe0 sp=0x14197fac5f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fac5fe8 sp=0x14197fac5fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 184 gp=0x14197fd3da40 m=nil [chan receive]:\nruntime.gopark(0x14197f3c29a0?, 0x2034990?, 0x10?, 0xea?, 0x59f0ad0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14198011f9d8 sp=0x14198011f9b8 pc=0x1f38d2e\nruntime.chanrecv(0x14197f3fa700, 0x14198011fcb8, 0x1)\n\t/usr/lib/golang/src/runtime/chan.go:667 +0x4ae fp=0x14198011fa50 sp=0x14198011f9d8 pc=0x1ec7d2e\nruntime.chanrecv2(0x5a09ed0?, 0x14197f530f90?)\n\t/usr/lib/golang/src/runtime/chan.go:514 +0x12 fp=0x14198011fa78 sp=0x14198011fa50 pc=0x1ec7872\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197fc4b340}, 0x14197f1c02a0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:141 +0x94d fp=0x14198011fd60 sp=0x14198011fa78 pc=0x391bb4d\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x74666968736e6570?, {0x59f0b40?, 0x14197fdd8140?}, {0x5a00300?, 0x14197fc4b340?}, 0x6d736f63222c2273?)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x14198011fe88 sp=0x14198011fd60 pc=0x3929c16\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14198011ff78 sp=0x14198011fe88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14198011ffe0 sp=0x14198011ff78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14198011ffe8 sp=0x14198011ffe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 135 gp=0x14197fb465a0 m=nil [semacquire]:\nruntime.gopark(0x14197f454000?, 0x22dc6c0?, 0xa0?, 0x82?, 0x1419808322a0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fe99ce8 sp=0x14197fe99cc8 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fe99d50 sp=0x14197fe99ce8 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x14197fe99df0 sp=0x14197fe99d50 pc=0x1edbb12\nruntime.gcAssistAlloc(0x14197fb465a0)\n\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x14197fe99e80 sp=0x14197fe99df0 pc=0x1ee06b6\nruntime.deductAssistCredit(0x0?)\n\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x14197fe99ea0 sp=0x14197fe99e80 pc=0x1ed10a6\nruntime.mallocgc(0x70, 0x0, 0x1)\n\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x14197fe99ed0 sp=0x14197fe99ea0 pc=0x1f35e07\nruntime.makechan(0x0?, 0x0)\n\t/usr/lib/golang/src/runtime/chan.go:99 +0x126 fp=0x14197fe99f10 sp=0x14197fe99ed0 pc=0x1ec6826\nnet/http.(*http2clientStream).cleanupWriteRequest(0x14197f2f1200, {0x0, 0x0})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8951 +0xd6 fp=0x14197fe99f90 sp=0x14197fe99f10 pc=0x22db1b6\nnet/http.(*http2clientStream).doRequest(0x14197f2f1200, 0x14197f7480a0?, 0x14197f4d0020?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8719 +0x28 fp=0x14197fe99fb8 sp=0x14197fe99f90 pc=0x22d99e8\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197fe99fe0 sp=0x14197fe99fb8 pc=0x22d999f\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fe99fe8 sp=0x14197fe99fe0 pc=0x1f40801\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 110\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n\ngoroutine 90 gp=0x14197fd82b40 m=nil [select]:\nruntime.gopark(0x14197ff1df08?, 0x6?, 0x40?, 0x2b?, 0x14197ff1dda4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f203c08 sp=0x14197f203be8 pc=0x1f38d2e\nruntime.selectgo(0x14197f203f08, 0x14197ff1dd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f203d38 sp=0x14197f203c08 pc=0x1f152a5\nnet/http.(*http2clientStream).writeRequest(0x14197f2f0f00, 0x14197fb50140, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x14197f203f90 sp=0x14197f203d38 pc=0x22da6a8\nnet/http.(*http2clientStream).doRequest(0x14197f2f0f00, 0x0?, 0x14197f1f1f98?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x14197f203fb8 sp=0x14197f203f90 pc=0x22d99d8\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197f203fe0 sp=0x14197f203fb8 pc=0x22d999f\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f203fe8 sp=0x14197f203fe0 pc=0x1f40801\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 183\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n\ngoroutine 225 gp=0x14197fd834a0 m=nil [sync.Cond.Wait]:\nruntime.gopark(0x22b8a73?, 0x141980390000?, 0x0?, 0x60?, 0xe?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419801298f8 sp=0x1419801298d8 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nsync.runtime_notifyListWait(0x14197f2f0f48, 0x3c)\n\t/usr/lib/golang/src/runtime/sema.go:617 +0x1b3 fp=0x141980129948 sp=0x1419801298f8 pc=0x1f3a8f3\nsync.(*Cond).Wait(0x14197fbb1640?)\n\t/usr/lib/golang/src/sync/cond.go:71 +0x73 fp=0x141980129980 sp=0x141980129948 pc=0x1f4d673\nnet/http.(*http2pipe).Read(0x14197f2f0f30, {0x14197fd98000, 0x1000, 0x1000})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:3928 +0xdb fp=0x1419801299e8 sp=0x141980129980 pc=0x22c2c1b\nnet/http.http2transportResponseBody.Read({0x141980129a60?}, {0x14197fd98000?, 0x20a6c63?, 0x1f3ec12?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9852 +0x59 fp=0x141980129a70 sp=0x1419801299e8 pc=0x22dfdd9\nbufio.(*Reader).fill(0x14197fc90180)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x141980129aa8 sp=0x141980129a70 pc=0x20a6c63\nbufio.(*Reader).ReadByte(0x14197fc90180)\n\t/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x141980129ac0 sp=0x141980129aa8 pc=0x20a73c7\ncompress/flate.(*decompressor).huffSym(0x141980058608, 0x141980058638)\n\t/usr/lib/golang/src/compress/flate/inflate.go:720 +0x7e fp=0x141980129b10 sp=0x141980129ac0 pc=0x228893e\ncompress/flate.(*decompressor).huffmanBlock(0x82bed80?)\n\t/usr/lib/golang/src/compress/flate/inflate.go:495 +0x45 fp=0x141980129b90 sp=0x141980129b10 pc=0x2287605\ncompress/flate.(*decompressor).Read(0x141980058608, {0x14197fd99cea, 0x316, 0xcea?})\n\t/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x141980129bc0 sp=0x141980129b90 pc=0x2286dbb\ncompress/gzip.(*Reader).Read(0x14197f3d8848, {0x14197fd99cea, 0x316, 0x316})\n\t/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x141980129c50 sp=0x141980129bc0 pc=0x228c838\ngithub.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x141980129cf0?, {0x14197fd99cea?, 0x1000?, 0x0?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x141980129c80 sp=0x141980129c50 pc=0x38b33e2\nbufio.(*Reader).fill(0x14197f546110)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x141980129cb8 sp=0x141980129c80 pc=0x20a6c63\nbufio.(*Reader).ReadSlice(0x14197f546110, 0xa)\n\t/usr/lib/golang/src/bufio/bufio.go:377 +0x29 fp=0x141980129d00 sp=0x141980129cb8 pc=0x20a7769\nbufio.(*Reader).collectFragments(0x14197f546110, 0xa)\n\t/usr/lib/golang/src/bufio/bufio.go:452 +0x85 fp=0x141980129de0 sp=0x141980129d00 pc=0x20a7b85\nbufio.(*Reader).ReadBytes(0x4bbc5a0?, 0xa0?)\n\t/usr/lib/golang/src/bufio/bufio.go:479 +0x17 fp=0x141980129e60 sp=0x141980129de0 pc=0x20a7e77\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.(*frameReader).advance(0x14197f546100)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:64 +0x45 fp=0x141980129eb0 sp=0x141980129e60 pc=0x38d6d45\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readRoutine(0x14197f546100, 0x14197f8c54f0)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:85 +0x51 fp=0x141980129fc0 sp=0x141980129eb0 pc=0x38d7591\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap2()\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x1b fp=0x141980129fe0 sp=0x141980129fc0 pc=0x38d74db\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x141980129fe8 sp=0x141980129fe0 pc=0x1f40801\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 183\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x24a\n\ngoroutine 174 gp=0x14197ff64000 m=nil [select]:\nruntime.gopark(0x1419800b9df8?, 0x2?, 0x0?, 0x0?, 0x1419800b9db4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf3c50 sp=0x14197fcf3c30 pc=0x1f38d2e\nruntime.selectgo(0x14197fcf3df8, 0x1419800b9db0, 0x59d87c8?, 0x0, 0x14197f2980e0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197fcf3d80 sp=0x14197fcf3c50 pc=0x1f152a5\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.nextFrame(0x14197f8c8370)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:198 +0x96 fp=0x14197fcf3e38 sp=0x14197fcf3d80 pc=0x38d7d16\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readPrimaryTable(0x14197f8c8370, 0x14197fb50000)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:258 +0xb1 fp=0x14197fcf3ec8 sp=0x14197fcf3e38 pc=0x38d8171\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x14197f8c8370)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:174 +0x189 fp=0x14197fcf3f30 sp=0x14197fcf3ec8 pc=0x38d7b89\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x14197f8c8370, 0x14197f59aee0)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x14197fcf3fc0 sp=0x14197fcf3f30 pc=0x38d788c\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x14197fcf3fe0 sp=0x14197fcf3fc0 pc=0x38d751b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf3fe8 sp=0x14197fcf3fe0 pc=0x1f40801\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 184\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc\n\nrax    0x0\nrbx    0x7f808bfff640\nrcx    0x7f80df60594c\nrdx    0x6\nrdi    0x109fd\nrsi    0x10a00\nrbp    0x10a00\nrsp    0x7f808bffe790\nr8     0x7f808bffe860\nr9     0x7f80df72b4e0\nr10    0x8\nr11    0x246\nr12    0x6\nr13    0x7f808bffeb60\nr14    0x14197f1334a0\nr15    0xffffffffffffffff\nrip    0x7f80df60594c\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\n\n  query-infra-svc: command failed: exit status 2\noutput: runtime/cgo: pthread_create failed: Resource temporarily unavailable\nSIGABRT: abort\nPC=0x7f7ef9baa94c m=3 sigcode=18446744073709551610\n\ngoroutine 0 gp=0x170ac5c1f0e0 m=3 mp=0x170ac5c91808 [idle]:\nruntime: g 0 gp=0x170ac5c1f0e0: unknown pc 0x7f7ef9baa94c\nstack: frame={sp:0x7f7eabffe790, fp:0x0} stack=[0x7f7eab7ff1e0,0x7f7eabffede0)\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n00007f7eabffe690: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6a0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6b0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6c0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6d0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6e0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6f0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe700: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe710: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe720: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe730: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe740: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe750: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe760: 00000000 00000000  00007f7e f9bb7fa9  ............~...\n00007f7eabffe770: 00000000 00000000  00000000 00000011  ................\n00007f7eabffe780: 00000000 00000130  00007f7e f9baa93e  0.......>...~...\n00007f7eabffe790: 00000000 00000120  00000000 00000000   ...............\n                  ^ SP\n00007f7eabffe7a0: 00000000 00000013  00000000 00000000  ................\n00007f7eabffe7b0: 00000000 00000000  00007f7e f9c2dcff  ............~...\n00007f7eabffe7c0: 00007f7e 7bfff910  00007f7e f9c2dcb1  ...{~.......~...\n00007f7eabffe7d0: 00007f7e 7bfff640  00007f7e abffeaa0  @..{~.......~...\n00007f7eabffe7e0: 00007f7e abffe90e  00007f7e abffe90f  ....~.......~...\n00007f7eabffe7f0: 00007f7e 7bfff640  00007f7e f9ba87f5  @..{~.......~...\n00007f7eabffe800: 00007f7e a4000020  00000000 00000120   ...~... .......\n00007f7eabffe810: 00000000 003d0f00  176a34c4 bac87500  ..=......u...4j.\n00007f7eabffe820: 00007f7e abfff640  00000000 00000006  @...~...........\n00007f7eabffe830: 00007f7e a4000f30  00007f7e abffeb60  0...~...`...~...\n00007f7eabffe840: 0000170a c5c1f0e0  00007f7e f9b5d646  ........F...~...\n00007f7eabffe850: 00007f7e f9d1ae90  00007f7e f9b477f3  ....~....w..~...\n00007f7eabffe860: 00000000 00000020  176a34c4 bac87500   ........u...4j.\n00007f7eabffe870: 00007f7e 7b7ff000  00007f7e f9c1cac4  ...{~.......~...\n00007f7eabffe880: 00000000 00000000  00000000 0000000d  ................\nruntime: g 0 gp=0x170ac5c1f0e0: unknown pc 0x7f7ef9baa94c\nstack: frame={sp:0x7f7eabffe790, fp:0x0} stack=[0x7f7eab7ff1e0,0x7f7eabffede0)\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n00007f7eabffe690: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6a0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6b0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6c0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6d0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6e0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6f0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe700: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe710: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe720: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe730: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe740: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe750: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe760: 00000000 00000000  00007f7e f9bb7fa9  ............~...\n00007f7eabffe770: 00000000 00000000  00000000 00000011  ................\n00007f7eabffe780: 00000000 00000130  00007f7e f9baa93e  0.......>...~...\n00007f7eabffe790: 00000000 00000120  00000000 00000000   ...............\n                  ^ SP\n00007f7eabffe7a0: 00000000 00000013  00000000 00000000  ................\n00007f7eabffe7b0: 00000000 00000000  00007f7e f9c2dcff  ............~...\n00007f7eabffe7c0: 00007f7e 7bfff910  00007f7e f9c2dcb1  ...{~.......~...\n00007f7eabffe7d0: 00007f7e 7bfff640  00007f7e abffeaa0  @..{~.......~...\n00007f7eabffe7e0: 00007f7e abffe90e  00007f7e abffe90f  ....~.......~...\n00007f7eabffe7f0: 00007f7e 7bfff640  00007f7e f9ba87f5  @..{~.......~...\n00007f7eabffe800: 00007f7e a4000020  00000000 00000120   ...~... .......\n00007f7eabffe810: 00000000 003d0f00  176a34c4 bac87500  ..=......u...4j.\n00007f7eabffe820: 00007f7e abfff640  00000000 00000006  @...~...........\n00007f7eabffe830: 00007f7e a4000f30  00007f7e abffeb60  0...~...`...~...\n00007f7eabffe840: 0000170a c5c1f0e0  00007f7e f9b5d646  ........F...~...\n00007f7eabffe850: 00007f7e f9d1ae90  00007f7e f9b477f3  ....~....w..~...\n00007f7eabffe860: 00000000 00000020  176a34c4 bac87500   ........u...4j.\n00007f7eabffe870: 00007f7e 7b7ff000  00007f7e f9c1cac4  ...{~.......~...\n00007f7eabffe880: 00000000 00000000  00000000 0000000d  ................\n\ngoroutine 1 gp=0x170ac5c1e1e0 m=nil [sync.WaitGroup.Wait]:\nruntime.gopark(0x170ac5cb27b8?, 0x2022a4d?, 0x70?, 0x8e?, 0x170ac5cb27e8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cb27a0 sp=0x170ac5cb2780 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x170ac65f8210, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x170ac5cb2808 sp=0x170ac5cb27a0 pc=0x1f162b2\nsync.runtime_SemacquireWaitGroup(0x170ac6709950?, 0xe0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x170ac5cb2840 sp=0x170ac5cb2808 pc=0x1f3a66e\nsync.(*WaitGroup).Wait(0x170ac65f8208)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x170ac5cb2868 sp=0x170ac5cb2840 pc=0x1f4f445\ngolang.org/x/sync/errgroup.(*Group).Wait(0x170ac65f8200)\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x170ac5cb2888 sp=0x170ac5cb2868 pc=0x362507e\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0}, {0x170ac626a1e0, 0x6, 0x6})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x170ac5cb2948 sp=0x170ac5cb2888 pc=0x3927e05\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).gatherInfraLogs(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:595 +0xadd fp=0x170ac5cb2d30 sp=0x170ac5cb2948 pc=0x392799d\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:333 +0x533 fp=0x170ac5cb3908 sp=0x170ac5cb2d30 pc=0x3921bf3\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedInfraQueryOptions).RunInfra(0x170ac5c2c108, {0x59f0b08, 0x170ac64108d0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:83 +0x71f fp=0x170ac5cb3b30 sp=0x170ac5cb3908 pc=0x393887f\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawInfraQueryOptions).Run(0x0?, {0x59f0b08, 0x170ac64108d0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_options.go:109 +0x54 fp=0x170ac5cb3b58 sp=0x170ac5cb3b30 pc=0x3939134\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryInfraCommand.func1(0x170ac66f5508?, {0x170ac5d40960?, 0x4?, 0x553b0e2?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:45 +0x28 fp=0x170ac5cb3b80 sp=0x170ac5cb3b58 pc=0x3938128\ngithub.com/spf13/cobra.(*Command).execute(0x170ac66f5508, {0x170ac5e36100, 0xa, 0xa})\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x170ac5cb3d28 sp=0x170ac5cb3b80 pc=0x212e614\ngithub.com/spf13/cobra.(*Command).ExecuteC(0x170ac6228608)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x170ac5cb3e18 sp=0x170ac5cb3d28 pc=0x212efa5\ngithub.com/spf13/cobra.(*Command).Execute(...)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071\ngithub.com/spf13/cobra.(*Command).ExecuteContext(...)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064\nmain.main()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x170ac5cb3f48 sp=0x170ac5cb3e18 pc=0x47ddd6a\nruntime.main()\n\t/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x170ac5cb3fe0 sp=0x170ac5cb3f48 pc=0x1f01975\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cb3fe8 sp=0x170ac5cb3fe0 pc=0x1f40801\n\ngoroutine 2 gp=0x170ac5c1ed20 m=nil [force gc (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdefa8 sp=0x170ac5cdef88 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.forcegchelper()\n\t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x170ac5cdefe0 sp=0x170ac5cdefa8 pc=0x1f01c93\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdefe8 sp=0x170ac5cdefe0 pc=0x1f40801\ncreated by runtime.init.7 in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\n\ngoroutine 18 gp=0x170ac5d843c0 m=nil [GC sweep wait]:\nruntime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cda788 sp=0x170ac5cda768 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.bgsweep(0x170ac5d92000)\n\t/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x170ac5cda7c8 sp=0x170ac5cda788 pc=0x1ee9f11\nruntime.gcenable.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x170ac5cda7e0 sp=0x170ac5cda7c8 pc=0x1edadd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cda7e8 sp=0x170ac5cda7e0 pc=0x1f40801\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\n\ngoroutine 19 gp=0x170ac5d845a0 m=nil [GC scavenge wait]:\nruntime.gopark(0x10000?, 0x5981cd8?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdaf78 sp=0x170ac5cdaf58 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.(*scavengerState).park(0x82979c0)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x170ac5cdafa8 sp=0x170ac5cdaf78 pc=0x1ee77e9\nruntime.bgscavenge(0x170ac5d92000)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x170ac5cdafc8 sp=0x170ac5cdafa8 pc=0x1ee7d99\nruntime.gcenable.gowrap2()\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x170ac5cdafe0 sp=0x170ac5cdafc8 pc=0x1edad97\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdafe8 sp=0x170ac5cdafe0 pc=0x1f40801\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\n\ngoroutine 34 gp=0x170ac5e00000 m=nil [GOMAXPROCS updater (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e06788 sp=0x170ac5e06768 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.updateMaxProcsGoroutine()\n\t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x170ac5e067e0 sp=0x170ac5e06788 pc=0x1f10267\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e067e8 sp=0x170ac5e067e0 pc=0x1f40801\ncreated by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\n\ngoroutine 35 gp=0x170ac5e005a0 m=nil [finalizer wait]:\nruntime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x170ac5cde601?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cde620 sp=0x170ac5cde600 pc=0x1f38d2e\nruntime.runFinalizers()\n\t/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x170ac5cde7e0 sp=0x170ac5cde620 pc=0x1ed9cc7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cde7e8 sp=0x170ac5cde7e0 pc=0x1f40801\ncreated by runtime.createfing in goroutine 1\n\t/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d\n\ngoroutine 36 gp=0x170ac5ff81e0 m=nil [cleanup wait]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e06f68 sp=0x170ac5e06f48 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.(*cleanupQueue).dequeue(0x82980e0)\n\t/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x170ac5e06fa0 sp=0x170ac5e06f68 pc=0x1ed6074\nruntime.runCleanups()\n\t/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x170ac5e06fe0 sp=0x170ac5e06fa0 pc=0x1ed66e5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e06fe8 sp=0x170ac5e06fe0 pc=0x1f40801\ncreated by runtime.(*cleanupQueue).createGs in goroutine 1\n\t/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5\n\ngoroutine 37 gp=0x170ac611bc20 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e07740 sp=0x170ac5e07720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5e077c8 sp=0x170ac5e07740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5e077e0 sp=0x170ac5e077c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e077e8 sp=0x170ac5e077e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 3 gp=0x170ac5c1f680 m=nil [runnable]:\nruntime.gopark(0x125c26559f5b?, 0x3?, 0x94?, 0x85?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdf740 sp=0x170ac5cdf720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdf7c8 sp=0x170ac5cdf740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdf7e0 sp=0x170ac5cdf7c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdf7e8 sp=0x170ac5cdf7e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 20 gp=0x170ac5d84780 m=nil [GC worker (idle)]:\nruntime.gopark(0x125c265c3db1?, 0x3?, 0x92?, 0xcd?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdb740 sp=0x170ac5cdb720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdb7c8 sp=0x170ac5cdb740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdb7e0 sp=0x170ac5cdb7c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdb7e8 sp=0x170ac5cdb7e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 38 gp=0x170ac6134000 m=nil [GC worker (idle)]:\nruntime.gopark(0x125c26588b9e?, 0x1?, 0xf7?, 0xb4?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e07f40 sp=0x170ac5e07f20 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5e07fc8 sp=0x170ac5e07f40 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5e07fe0 sp=0x170ac5e07fc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e07fe8 sp=0x170ac5e07fe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 4 gp=0x170ac5c1f860 m=10 mp=0x170ac6280008 [GC mark termination]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x170ac5cdfea0 sp=0x170ac5cdfe90 pc=0x1f3ebc8\nruntime.forEachP(...)\n\t/usr/lib/golang/src/runtime/proc.go:2112\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x170ac5cdff40 sp=0x170ac5cdfea0 pc=0x1edbc34\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x170ac5cdffc8 sp=0x170ac5cdff40 pc=0x1edda88\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdffe0 sp=0x170ac5cdffc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdffe8 sp=0x170ac5cdffe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 21 gp=0x170ac5d84960 m=nil [GC worker (idle)]:\nruntime.gopark(0x125c265818cf?, 0x3?, 0xa5?, 0x9c?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdbf40 sp=0x170ac5cdbf20 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdbfc8 sp=0x170ac5cdbf40 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdbfe0 sp=0x170ac5cdbfc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdbfe8 sp=0x170ac5cdbfe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 22 gp=0x170ac5d84b40 m=nil [GC worker (idle)]:\nruntime.gopark(0x125c265846e7?, 0x3?, 0x77?, 0xb6?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdc740 sp=0x170ac5cdc720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdc7c8 sp=0x170ac5cdc740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdc7e0 sp=0x170ac5cdc7c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdc7e8 sp=0x170ac5cdc7e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 23 gp=0x170ac5d84d20 m=nil [GC worker (idle)]:\nruntime.gopark(0x125c26581951?, 0x3?, 0x91?, 0x58?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdcf40 sp=0x170ac5cdcf20 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdcfc8 sp=0x170ac5cdcf40 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdcfe0 sp=0x170ac5cdcfc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdcfe8 sp=0x170ac5cdcfe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 39 gp=0x170ac648a3c0 m=nil [select, locked to thread]:\nruntime.gopark(0x170ac5e097a8?, 0x2?, 0x1?, 0xa3?, 0x170ac5e09794?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e09628 sp=0x170ac5e09608 pc=0x1f38d2e\nruntime.selectgo(0x170ac5e097a8, 0x170ac5e09790, 0x0?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5e09758 sp=0x170ac5e09628 pc=0x1f152a5\nruntime.ensureSigM.func1()\n\t/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x170ac5e097e0 sp=0x170ac5e09758 pc=0x1f328e8\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e097e8 sp=0x170ac5e097e0 pc=0x1f40801\ncreated by runtime.ensureSigM in goroutine 1\n\t/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5\n\ngoroutine 24 gp=0x170ac62f4960 m=8 mp=0x170ac6180008 [syscall]:\nruntime.notetsleepg(0x82bdea0, 0xffffffffffffffff)\n\t/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x170ac5e037a0 sp=0x170ac5e03778 pc=0x1ece509\nos/signal.signal_recv()\n\t/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x170ac5e037c0 sp=0x170ac5e037a0 pc=0x1f3b298\nos/signal.loop()\n\t/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x170ac5e037e0 sp=0x170ac5e037c0 pc=0x2030553\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e037e8 sp=0x170ac5e037e0 pc=0x1f40801\ncreated by os/signal.Notify.func1.1 in goroutine 1\n\t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\n\ngoroutine 50 gp=0x170ac6135a40 m=nil [select]:\nruntime.gopark(0x170ac66e3798?, 0x2?, 0x80?, 0x6f?, 0x170ac66e378c?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac66e3628 sp=0x170ac66e3608 pc=0x1f38d2e\nruntime.selectgo(0x170ac66e3798, 0x170ac66e3788, 0x0?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac66e3758 sp=0x170ac66e3628 pc=0x1f152a5\nos/signal.NotifyContext.func1()\n\t/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x170ac66e37e0 sp=0x170ac66e3758 pc=0x202ff9c\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac66e37e8 sp=0x170ac66e37e0 pc=0x1f40801\ncreated by os/signal.NotifyContext in goroutine 1\n\t/usr/lib/golang/src/os/signal/signal.go:291 +0x158\n\ngoroutine 102 gp=0x170ac61843c0 m=nil [select]:\nruntime.gopark(0x170ac5cefc70?, 0x2?, 0xc8?, 0xf9?, 0x170ac5cefb6c?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cef9f0 sp=0x170ac5cef9d0 pc=0x1f38d2e\nruntime.selectgo(0x170ac5cefc70, 0x170ac5cefb68, 0x554857c?, 0x0, 0x226?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5cefb20 sp=0x170ac5cef9f0 pc=0x1f152a5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x170ac5c49130}, 0x170ac65fa070, 0x170ac6410cf0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:283 +0x21b fp=0x170ac5ceff48 sp=0x170ac5cefb20 pc=0x39204fb\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x170ac5ceff78 sp=0x170ac5ceff48 pc=0x392800c\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5ceffe0 sp=0x170ac5ceff78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5ceffe8 sp=0x170ac5ceffe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 104 gp=0x170ac6184d20 m=nil [select]:\nruntime.gopark(0x170ac6992ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69929d8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6992848 sp=0x170ac6992828 pc=0x1f38d2e\nruntime.selectgo(0x170ac6992ac8, 0x170ac69929d0, 0x55562e1?, 0x0, 0x1f6d6cf?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6992978 sp=0x170ac6992848 pc=0x1f152a5\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac619fb80, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac6992b40 sp=0x170ac6992978 pc=0x22d92e5\nnet/http.(*http2ClientConn).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac619fb80, {0x4c?, 0xfa?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac6992c80 sp=0x170ac6992b40 pc=0x22d50ad\nnet/http.(*http2Transport).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac619fb80?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac6992cc0 sp=0x170ac6992c80 pc=0x22e3a9c\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac619fa40)\n\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac6992ea0 sp=0x170ac6992cc0 pc=0x231037f\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac6992ec0 sp=0x170ac6992ea0 pc=0x2321178\nnet/http.send(0x170ac619fa40, {0x59a1ce0, 0x8175400}, {0x170ac6993138?, 0x1f3bac6?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69930b0 sp=0x170ac6992ec0 pc=0x22ad9ab\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac619fa40, {0x170ac69931e8?, 0x1fdd787?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac6993148 sp=0x170ac69930b0 pc=0x22ad1d8\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac619fa40)\n\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac6993338 sp=0x170ac6993148 pc=0x22af817\nnet/http.(*Client).Do(...)\n\t/usr/lib/golang/src/net/http/client.go:592\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5c49450}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e34a0}, 0x170ac65f7ce0, {0x170ac65d9560, 0x112})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac6993630 sp=0x170ac6993338 pc=0x38de6bd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5c49450}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x170ac65f7c80, 0x0, {0x0, ...}, ...})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac6993888 sp=0x170ac6993630 pc=0x38ddecd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5c49450?}, 0xa8?, {0x170ac5db1a50?, 0x170ac6764a48?}, 0x3?, 0x2022a4d?)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac6993960 sp=0x170ac6993888 pc=0x38dd40b\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194700?}, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x170ac6764a48, 0x3, 0x4})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69939e0 sp=0x170ac6993960 pc=0x38e51e2\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194700}, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x0?, 0x2?, 0x1?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac6993a78 sp=0x170ac69939e0 pc=0x38e4eaf\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020500}, 0x170ac65fa000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac6993d60 sp=0x170ac6993a78 pc=0x391b5c5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac5fe1880?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020500?}, 0x0?)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac6993e88 sp=0x170ac6993d60 pc=0x3929c16\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac6993f78 sp=0x170ac6993e88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac6993fe0 sp=0x170ac6993f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6993fe8 sp=0x170ac6993fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 101 gp=0x170ac61d4000 m=nil [sync.WaitGroup.Wait]:\nruntime.gopark(0x170ac5de7e00?, 0x1ed0765?, 0xb0?, 0xfa?, 0xe?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de7da0 sp=0x170ac5de7d80 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x170ac65f8290, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x170ac5de7e08 sp=0x170ac5de7da0 pc=0x1f162b2\nsync.runtime_SemacquireWaitGroup(0x170ac6709a70?, 0x0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x170ac5de7e40 sp=0x170ac5de7e08 pc=0x1f3a66e\nsync.(*WaitGroup).Wait(0x170ac65f8288)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x170ac5de7e68 sp=0x170ac5de7e40 pc=0x1f4f445\ngolang.org/x/sync/errgroup.(*Group).Wait(0x170ac65f8280)\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x170ac5de7e88 sp=0x170ac5de7e68 pc=0x362507e\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x170ac5fe1140, {0x59f0b40, 0x170ac5c49130}, {0x170ac626a1e0, 0x6, 0x170ac5cac1c0?}, 0x170ac65fa000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x170ac5de7f00 sp=0x170ac5de7e88 pc=0x392976a\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x170ac5de7f78 sp=0x170ac5de7f00 pc=0x392808e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5de7fe0 sp=0x170ac5de7f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de7fe8 sp=0x170ac5de7fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 67 gp=0x170ac62f5a40 m=nil [IO wait]:\nruntime.gopark(0x170ac6497780?, 0x8?, 0x40?, 0x16?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de47b8 sp=0x170ac5de4798 pc=0x1f38d2e\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x170ac5de47f0 sp=0x170ac5de47b8 pc=0x1ef9e57\ninternal/poll.runtime_pollWait(0x7f7eb1670000, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x170ac5de4810 sp=0x170ac5de47f0 pc=0x1f375c5\ninternal/poll.(*pollDesc).wait(0x170ac620b480?, 0x170ac65a2000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x170ac5de4838 sp=0x170ac5de4810 pc=0x1fbd167\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0x170ac620b480, {0x170ac65a2000, 0x3500, 0x3500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x170ac5de48d0 sp=0x170ac5de4838 pc=0x1fbe4ce\nnet.(*netFD).Read(0x170ac620b480, {0x170ac65a2000?, 0x170ac5de4901?, 0x22a2e75?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x170ac5de4918 sp=0x170ac5de48d0 pc=0x20798c5\nnet.(*conn).Read(0x170ac62823d8, {0x170ac65a2000?, 0x7f7eb047fc00?, 0x7f7ef9af2e00?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x170ac5de4960 sp=0x170ac5de4918 pc=0x208b7a5\ncrypto/tls.(*atLeastReader).Read(0x170ac6708f30, {0x170ac65a2000?, 0x170ac5de4a48?, 0x1efcc28?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x170ac5de49a8 sp=0x170ac5de4960 pc=0x2222ddb\nbytes.(*Buffer).ReadFrom(0x170ac605fb28, {0x59a3680, 0x170ac6708f30})\n\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x170ac5de4a00 sp=0x170ac5de49a8 pc=0x1fe9258\ncrypto/tls.(*Conn).readFromUntil(0x170ac605f888, {0x59a32c0, 0x170ac62823d8}, 0x222764c?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x170ac5de4a38 sp=0x170ac5de4a00 pc=0x2222fbe\ncrypto/tls.(*Conn).readRecordOrCCS(0x170ac605f888, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x170ac5de4c18 sp=0x170ac5de4a38 pc=0x222009b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:588\ncrypto/tls.(*Conn).Read(0x170ac605f888, {0x170ac6029000, 0x1000, 0x22c34e0?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x170ac5de4c80 sp=0x170ac5de4c18 pc=0x22267c5\nbufio.(*Reader).Read(0x170ac647aa20, {0x170ac6153624, 0x9, 0x22e100e?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0x170ac5de4cb8 sp=0x170ac5de4c80 pc=0x20a71f7\nio.ReadAtLeast({0x59a1260, 0x170ac647aa20}, {0x170ac6153624, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0x170ac5de4d00 sp=0x170ac5de4cb8 pc=0x1fb17ee\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\nnet/http.http2readFrameHeader({0x170ac6153624, 0x9, 0x170a00000005?}, {0x59a1260?, 0x170ac647aa20?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:1805 +0x65 fp=0x170ac5de4d50 sp=0x170ac5de4d00 pc=0x22b9e25\nnet/http.(*http2Framer).ReadFrameHeader(0x170ac61535e0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:2071 +0x6b fp=0x170ac5de4dd8 sp=0x170ac5de4d50 pc=0x22ba9eb\nnet/http.(*http2Framer).ReadFrame(0x170ac61535e0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:2130 +0x18 fp=0x170ac5de4e18 sp=0x170ac5de4dd8 pc=0x22bb1b8\nnet/http.(*http2clientConnReadLoop).run(0x170ac5de4fa8)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9550 +0xca fp=0x170ac5de4f60 sp=0x170ac5de4e18 pc=0x22de14a\nnet/http.(*http2ClientConn).readLoop(0x170ac61d9880)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x170ac5de4fc8 sp=0x170ac5de4f60 pc=0x22dd532\nnet/http.(*http2Transport).newClientConn.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x170ac5de4fe0 sp=0x170ac5de4fc8 pc=0x22d7057\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de4fe8 sp=0x170ac5de4fe0 pc=0x1f40801\ncreated by net/http.(*http2Transport).newClientConn in goroutine 66\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5\n\ngoroutine 73 gp=0x170ac648ad20 m=nil [IO wait]:\nruntime.gopark(0x170ac6497e40?, 0x1edb176?, 0x80?, 0xed?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de6808 sp=0x170ac5de67e8 pc=0x1f38d2e\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x170ac5de6840 sp=0x170ac5de6808 pc=0x1ef9e57\ninternal/poll.runtime_pollWait(0x7f7eb166fe00, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x170ac5de6860 sp=0x170ac5de6840 pc=0x1f375c5\ninternal/poll.(*pollDesc).wait(0x170ac60c4e80?, 0x170ac5ecd300?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x170ac5de6888 sp=0x170ac5de6860 pc=0x1fbd167\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0x170ac60c4e80, {0x170ac5ecd300, 0x1300, 0x1300})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x170ac5de6920 sp=0x170ac5de6888 pc=0x1fbe4ce\nnet.(*netFD).Read(0x170ac60c4e80, {0x170ac5ecd300?, 0x170ac5ecd30a?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x170ac5de6968 sp=0x170ac5de6920 pc=0x20798c5\nnet.(*conn).Read(0x170ac5d82278, {0x170ac5ecd300?, 0x7f7eb047fc00?, 0x7f7ef9af2e00?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x170ac5de69b0 sp=0x170ac5de6968 pc=0x208b7a5\ncrypto/tls.(*atLeastReader).Read(0x170ac67088e8, {0x170ac5ecd300?, 0x170ac5de6a98?, 0x1efcc28?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x170ac5de69f8 sp=0x170ac5de69b0 pc=0x2222ddb\nbytes.(*Buffer).ReadFrom(0x170ac5dd02a8, {0x59a3680, 0x170ac67088e8})\n\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x170ac5de6a50 sp=0x170ac5de69f8 pc=0x1fe9258\ncrypto/tls.(*Conn).readFromUntil(0x170ac5dd0008, {0x59a32c0, 0x170ac5d82278}, 0x222764c?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x170ac5de6a88 sp=0x170ac5de6a50 pc=0x2222fbe\ncrypto/tls.(*Conn).readRecordOrCCS(0x170ac5dd0008, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x170ac5de6c68 sp=0x170ac5de6a88 pc=0x222009b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:588\ncrypto/tls.(*Conn).Read(0x170ac5dd0008, {0x170ac6564000, 0x1000, 0x4e55540?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x170ac5de6cd0 sp=0x170ac5de6c68 pc=0x22267c5\nnet/http.(*persistConn).Read(0x170ac5f73180, {0x170ac6564000?, 0x59a0ea0?, 0x81260e0?})\n\t/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x170ac5de6d30 sp=0x170ac5de6cd0 pc=0x2319d07\nbufio.(*Reader).fill(0x170ac65537a0)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x170ac5de6d68 sp=0x170ac5de6d30 pc=0x20a6c63\nbufio.(*Reader).Peek(0x170ac65537a0, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x170ac5de6d80 sp=0x170ac5de6d68 pc=0x20a6df3\nnet/http.(*persistConn).readLoop(0x170ac5f73180)\n\t/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x170ac5de6fc8 sp=0x170ac5de6d80 pc=0x231a812\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x170ac5de6fe0 sp=0x170ac5de6fc8 pc=0x2319197\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de6fe8 sp=0x170ac5de6fe0 pc=0x1f40801\ncreated by net/http.(*Transport).dialConn in goroutine 82\n\t/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac\n\ngoroutine 103 gp=0x170ac648af00 m=nil [select]:\nruntime.gopark(0x170ac5de0d00?, 0x2?, 0xa0?, 0x6a?, 0x170ac5de0c74?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de0ac8 sp=0x170ac5de0aa8 pc=0x1f38d2e\nruntime.selectgo(0x170ac5de0d00, 0x170ac5de0c70, 0x553bfdc?, 0x0, 0x5a48c80?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5de0bf8 sp=0x170ac5de0ac8 pc=0x1f152a5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x0?, {0x59f0b40, 0x170ac5c49130}, 0x170ac65fa000, 0x170ac65fa070)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x170ac5de0f10 sp=0x170ac5de0bf8 pc=0x392836e\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x170ac5de0f78 sp=0x170ac5de0f10 pc=0x3927f3d\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5de0fe0 sp=0x170ac5de0f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de0fe8 sp=0x170ac5de0fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 74 gp=0x170ac648b4a0 m=nil [select]:\nruntime.gopark(0x170ac5df1f48?, 0x2?, 0x0?, 0xb6?, 0x170ac5df1ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6760d88 sp=0x170ac6760d68 pc=0x1f38d2e\nruntime.selectgo(0x170ac6760f48, 0x170ac5df1ef0, 0x170ac68d6980?, 0x0, 0x170ac655e8d0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6760eb8 sp=0x170ac6760d88 pc=0x1f152a5\nnet/http.(*persistConn).writeLoop(0x170ac5f73180)\n\t/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x170ac6760fc8 sp=0x170ac6760eb8 pc=0x231c426\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x170ac6760fe0 sp=0x170ac6760fc8 pc=0x2319157\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6760fe8 sp=0x170ac6760fe0 pc=0x1f40801\ncreated by net/http.(*Transport).dialConn in goroutine 82\n\t/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe\n\ngoroutine 105 gp=0x170ac61d43c0 m=nil [select]:\nruntime.gopark(0x170ac699cac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac699c9d8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac699c848 sp=0x170ac699c828 pc=0x1f38d2e\nruntime.selectgo(0x170ac699cac8, 0x170ac699c9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac699c978 sp=0x170ac699c848 pc=0x1f152a5\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e000, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac699cb40 sp=0x170ac699c978 pc=0x22d92e5\nnet/http.(*http2ClientConn).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e000, {0x4c?, 0xfa?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac699cc80 sp=0x170ac699cb40 pc=0x22d50ad\nnet/http.(*http2Transport).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e000?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac699ccc0 sp=0x170ac699cc80 pc=0x22e3a9c\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac619fcc0)\n\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac699cea0 sp=0x170ac699ccc0 pc=0x231037f\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac699cec0 sp=0x170ac699cea0 pc=0x2321178\nnet/http.send(0x170ac619fcc0, {0x59a1ce0, 0x8175400}, {0x170ac699d138?, 0x1f3bac6?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac699d0b0 sp=0x170ac699cec0 pc=0x22ad9ab\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac619fcc0, {0x170ac699d1e8?, 0x1fdd787?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac699d148 sp=0x170ac699d0b0 pc=0x22ad1d8\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac619fcc0)\n\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac699d338 sp=0x170ac699d148 pc=0x22af817\nnet/http.(*Client).Do(...)\n\t/usr/lib/golang/src/net/http/client.go:592\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e000}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e36b0}, 0x170ac65f7e00, {0x170ac65d9680, 0x118})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac699d630 sp=0x170ac699d338 pc=0x38de6bd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e000}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x170ac65f7da0, 0x0, {0x0, ...}, ...})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac699d888 sp=0x170ac699d630 pc=0x38ddecd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e000?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de1a48?}, 0x3?, 0x2022a4d?)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac699d960 sp=0x170ac699d888 pc=0x38dd40b\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194850?}, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x170ac5de1a48, 0x3, 0x4})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac699d9e0 sp=0x170ac699d960 pc=0x38e51e2\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194850}, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x0?, 0x2?, 0x0?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac699da78 sp=0x170ac699d9e0 pc=0x38e4eaf\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac60207c0}, 0x170ac65fa000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac699dd60 sp=0x170ac699da78 pc=0x391b5c5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac60207c0?}, 0x0?)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac699de88 sp=0x170ac699dd60 pc=0x3929c16\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac699df78 sp=0x170ac699de88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac699dfe0 sp=0x170ac699df78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac699dfe8 sp=0x170ac699dfe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 107 gp=0x170ac61d45a0 m=nil [select]:\nruntime.gopark(0x170ac69b2ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69b29d8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69b2848 sp=0x170ac69b2828 pc=0x1f38d2e\nruntime.selectgo(0x170ac69b2ac8, 0x170ac69b29d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69b2978 sp=0x170ac69b2848 pc=0x1f152a5\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e500, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69b2b40 sp=0x170ac69b2978 pc=0x22d92e5\nnet/http.(*http2ClientConn).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e500, {0x4c?, 0xfa?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69b2c80 sp=0x170ac69b2b40 pc=0x22d50ad\nnet/http.(*http2Transport).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e500?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69b2cc0 sp=0x170ac69b2c80 pc=0x22e3a9c\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac699e3c0)\n\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69b2ea0 sp=0x170ac69b2cc0 pc=0x231037f\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69b2ec0 sp=0x170ac69b2ea0 pc=0x2321178\nnet/http.send(0x170ac699e3c0, {0x59a1ce0, 0x8175400}, {0x170ac69b3138?, 0x1f3bac6?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69b30b0 sp=0x170ac69b2ec0 pc=0x22ad9ab\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac699e3c0, {0x4d3aa8?, 0xcd6610?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69b3148 sp=0x170ac69b30b0 pc=0x22ad1d8\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac699e3c0)\n\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69b3338 sp=0x170ac69b3148 pc=0x22af817\nnet/http.(*Client).Do(...)\n\t/usr/lib/golang/src/net/http/client.go:592\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e1e0}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3a60}, 0x170ac69a0120, {0x170ac65d98c0, 0x111})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69b3630 sp=0x170ac69b3338 pc=0x38de6bd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e1e0}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x170ac69a00c0, 0x0, {0x0, ...}, ...})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69b3888 sp=0x170ac69b3630 pc=0x38ddecd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e1e0?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de2a48?}, 0x3?, 0x2022a4d?)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69b3960 sp=0x170ac69b3888 pc=0x38dd40b\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194bd0?}, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x170ac5de2a48, 0x3, 0x4})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69b39e0 sp=0x170ac69b3960 pc=0x38e51e2\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194bd0}, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x0?, 0x2?, 0x0?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69b3a78 sp=0x170ac69b39e0 pc=0x38e4eaf\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020ec0}, 0x170ac65fa000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69b3d60 sp=0x170ac69b3a78 pc=0x391b5c5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020ec0?}, 0x0?)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69b3e88 sp=0x170ac69b3d60 pc=0x3929c16\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69b3f78 sp=0x170ac69b3e88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69b3fe0 sp=0x170ac69b3f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69b3fe8 sp=0x170ac69b3fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 106 gp=0x170ac648b680 m=nil [select]:\nruntime.gopark(0x170ac69aaac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69aa9d8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69aa848 sp=0x170ac69aa828 pc=0x1f38d2e\nruntime.selectgo(0x170ac69aaac8, 0x170ac69aa9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69aa978 sp=0x170ac69aa848 pc=0x1f152a5\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e280, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69aab40 sp=0x170ac69aa978 pc=0x22d92e5\nnet/http.(*http2ClientConn).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e280, {0x4c?, 0xfa?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69aac80 sp=0x170ac69aab40 pc=0x22d50ad\nnet/http.(*http2Transport).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e280?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69aacc0 sp=0x170ac69aac80 pc=0x22e3a9c\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac699e140)\n\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69aaea0 sp=0x170ac69aacc0 pc=0x231037f\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69aaec0 sp=0x170ac69aaea0 pc=0x2321178\nnet/http.send(0x170ac699e140, {0x59a1ce0, 0x8175400}, {0x170ac69ab138?, 0x1f3bac6?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69ab0b0 sp=0x170ac69aaec0 pc=0x22ad9ab\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac699e140, {0x170ac69ab1e8?, 0x1fdd787?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69ab148 sp=0x170ac69ab0b0 pc=0x22ad1d8\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac699e140)\n\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69ab338 sp=0x170ac69ab148 pc=0x22af817\nnet/http.(*Client).Do(...)\n\t/usr/lib/golang/src/net/http/client.go:592\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e0f0}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e38c0}, 0x170ac69a0030, {0x170ac65d97a0, 0x10e})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69ab630 sp=0x170ac69ab338 pc=0x38de6bd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e0f0}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x170ac65f7ec0, 0x0, {0x0, ...}, ...})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69ab888 sp=0x170ac69ab630 pc=0x38ddecd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e0f0?}, 0xa8?, {0x170ac5db1a50?, 0x170ac6766a48?}, 0x3?, 0x2022a4d?)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69ab960 sp=0x170ac69ab888 pc=0x38dd40b\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194a80?}, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x170ac6766a48, 0x3, 0x4})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69ab9e0 sp=0x170ac69ab960 pc=0x38e51e2\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194a80}, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x0?, 0x2?, 0x0?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69aba78 sp=0x170ac69ab9e0 pc=0x38e4eaf\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020cc0}, 0x170ac65fa000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69abd60 sp=0x170ac69aba78 pc=0x391b5c5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac6766ea0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020cc0?}, 0x170ac5f6f3b0?)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69abe88 sp=0x170ac69abd60 pc=0x3929c16\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69abf78 sp=0x170ac69abe88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69abfe0 sp=0x170ac69abf78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69abfe8 sp=0x170ac69abfe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 108 gp=0x170ac61d4780 m=nil [select]:\nruntime.gopark(0x170ac69c0ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69c09d8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69c0848 sp=0x170ac69c0828 pc=0x1f38d2e\nruntime.selectgo(0x170ac69c0ac8, 0x170ac69c09d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69c0978 sp=0x170ac69c0848 pc=0x1f152a5\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e780, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69c0b40 sp=0x170ac69c0978 pc=0x22d92e5\nnet/http.(*http2ClientConn).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e780, {0x4c?, 0xfa?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69c0c80 sp=0x170ac69c0b40 pc=0x22d50ad\nnet/http.(*http2Transport).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e780?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69c0cc0 sp=0x170ac69c0c80 pc=0x22e3a9c\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac699e640)\n\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69c0ea0 sp=0x170ac69c0cc0 pc=0x231037f\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69c0ec0 sp=0x170ac69c0ea0 pc=0x2321178\nnet/http.send(0x170ac699e640, {0x59a1ce0, 0x8175400}, {0x170ac69c1138?, 0x1f3bac6?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69c10b0 sp=0x170ac69c0ec0 pc=0x22ad9ab\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac699e640, {0x82bed80?, 0x6401fdd700?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69c1148 sp=0x170ac69c10b0 pc=0x22ad1d8\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac699e640)\n\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69c1338 sp=0x170ac69c1148 pc=0x22af817\nnet/http.(*Client).Do(...)\n\t/usr/lib/golang/src/net/http/client.go:592\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e280}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3bf0}, 0x170ac69a0210, {0x170ac65d99e0, 0x110})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69c1630 sp=0x170ac69c1338 pc=0x38de6bd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e280}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x170ac69a01b0, 0x0, {0x0, ...}, ...})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69c1888 sp=0x170ac69c1630 pc=0x38ddecd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e280?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de2a48?}, 0x3?, 0x2022a4d?)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69c1960 sp=0x170ac69c1888 pc=0x38dd40b\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194d90?}, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x170ac5de2a48, 0x3, 0x4})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69c19e0 sp=0x170ac69c1960 pc=0x38e51e2\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194d90}, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x0?, 0x2?, 0x0?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69c1a78 sp=0x170ac69c19e0 pc=0x38e4eaf\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6021d80}, 0x170ac65fa000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69c1d60 sp=0x170ac69c1a78 pc=0x391b5c5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac66e1d6a?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6021d80?}, 0x0?)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69c1e88 sp=0x170ac69c1d60 pc=0x3929c16\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69c1f78 sp=0x170ac69c1e88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69c1fe0 sp=0x170ac69c1f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69c1fe8 sp=0x170ac69c1fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 109 gp=0x170ac61d4960 m=nil [select]:\nruntime.gopark(0x170ac69caac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69ca9d8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69ca848 sp=0x170ac69ca828 pc=0x1f38d2e\nruntime.selectgo(0x170ac69caac8, 0x170ac69ca9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69ca978 sp=0x170ac69ca848 pc=0x1f152a5\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699ea00, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69cab40 sp=0x170ac69ca978 pc=0x22d92e5\nnet/http.(*http2ClientConn).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699ea00, {0x4c?, 0xfa?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69cac80 sp=0x170ac69cab40 pc=0x22d50ad\nnet/http.(*http2Transport).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699ea00?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69cacc0 sp=0x170ac69cac80 pc=0x22e3a9c\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac699e8c0)\n\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69caea0 sp=0x170ac69cacc0 pc=0x231037f\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69caec0 sp=0x170ac69caea0 pc=0x2321178\nnet/http.send(0x170ac699e8c0, {0x59a1ce0, 0x8175400}, {0x170ac69cb138?, 0x1f3bac6?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69cb0b0 sp=0x170ac69caec0 pc=0x22ad9ab\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac699e8c0, {0x170ac69cb1e8?, 0x1fdd787?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69cb148 sp=0x170ac69cb0b0 pc=0x22ad1d8\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac699e8c0)\n\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69cb338 sp=0x170ac69cb148 pc=0x22af817\nnet/http.(*Client).Do(...)\n\t/usr/lib/golang/src/net/http/client.go:592\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e410}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3d80}, 0x170ac69a0300, {0x170ac65d9b00, 0x114})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69cb630 sp=0x170ac69cb338 pc=0x38de6bd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e410}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x170ac69a02a0, 0x0, {0x0, ...}, ...})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69cb888 sp=0x170ac69cb630 pc=0x38ddecd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e410?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de3a48?}, 0x3?, 0x2022a4d?)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69cb960 sp=0x170ac69cb888 pc=0x38dd40b\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194620?}, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x170ac5de3a48, 0x3, 0x4})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69cb9e0 sp=0x170ac69cb960 pc=0x38e51e2\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194620}, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x0?, 0x2?, 0x2700000000000000?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69cba78 sp=0x170ac69cb9e0 pc=0x38e4eaf\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac65f81c0}, 0x170ac65fa000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69cbd60 sp=0x170ac69cba78 pc=0x391b5c5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac65f81c0?}, 0x0?)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69cbe88 sp=0x170ac69cbd60 pc=0x3929c16\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69cbf78 sp=0x170ac69cbe88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69cbfe0 sp=0x170ac69cbf78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69cbfe8 sp=0x170ac69cbfe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 110 gp=0x170ac61d4b40 m=nil [select]:\nruntime.gopark(0x170ac6927f08?, 0x6?, 0x40?, 0x4b?, 0x170ac6927da4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6927c08 sp=0x170ac6927be8 pc=0x1f38d2e\nruntime.selectgo(0x170ac6927f08, 0x170ac6927d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6927d38 sp=0x170ac6927c08 pc=0x1f152a5\nnet/http.(*http2clientStream).writeRequest(0x170ac61a0f00, 0x170ac619fb80, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6927f90 sp=0x170ac6927d38 pc=0x22da6a8\nnet/http.(*http2clientStream).doRequest(0x170ac61a0f00, 0x0?, 0x170ac66e2798?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6927fb8 sp=0x170ac6927f90 pc=0x22d99d8\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6927fe0 sp=0x170ac6927fb8 pc=0x22d999f\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6927fe8 sp=0x170ac6927fe0 pc=0x1f40801\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 104\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n\ngoroutine 111 gp=0x170ac61d4d20 m=nil [select]:\nruntime.gopark(0x170ac658df08?, 0x6?, 0x0?, 0x7?, 0x170ac658dda4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac658dc08 sp=0x170ac658dbe8 pc=0x1f38d2e\nruntime.selectgo(0x170ac658df08, 0x170ac658dd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac658dd38 sp=0x170ac658dc08 pc=0x1f152a5\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1080, 0x170ac699e000, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac658df90 sp=0x170ac658dd38 pc=0x22da6a8\nnet/http.(*http2clientStream).doRequest(0x170ac61a1080, 0x0?, 0x170ac66e2798?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac658dfb8 sp=0x170ac658df90 pc=0x22d99d8\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac658dfe0 sp=0x170ac658dfb8 pc=0x22d999f\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac658dfe8 sp=0x170ac658dfe0 pc=0x1f40801\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 105\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n\ngoroutine 112 gp=0x170ac61d4f00 m=nil [select]:\nruntime.gopark(0x170ac6589f08?, 0x6?, 0x0?, 0x7?, 0x170ac6589da4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6589c08 sp=0x170ac6589be8 pc=0x1f38d2e\nruntime.selectgo(0x170ac6589f08, 0x170ac6589d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6589d38 sp=0x170ac6589c08 pc=0x1f152a5\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1200, 0x170ac699e280, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6589f90 sp=0x170ac6589d38 pc=0x22da6a8\nnet/http.(*http2clientStream).doRequest(0x170ac61a1200, 0x0?, 0x170ac66e1f98?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6589fb8 sp=0x170ac6589f90 pc=0x22d99d8\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6589fe0 sp=0x170ac6589fb8 pc=0x22d999f\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6589fe8 sp=0x170ac6589fe0 pc=0x1f40801\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 106\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n\ngoroutine 113 gp=0x170ac61d50e0 m=nil [select]:\nruntime.gopark(0x170ac658ff08?, 0x6?, 0x0?, 0x7?, 0x170ac658fda4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac658fc08 sp=0x170ac658fbe8 pc=0x1f38d2e\nruntime.selectgo(0x170ac658ff08, 0x170ac658fd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac658fd38 sp=0x170ac658fc08 pc=0x1f152a5\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1380, 0x170ac699e780, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac658ff90 sp=0x170ac658fd38 pc=0x22da6a8\nnet/http.(*http2clientStream).doRequest(0x170ac61a1380, 0x0?, 0x170ac66e1f98?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac658ffb8 sp=0x170ac658ff90 pc=0x22d99d8\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac658ffe0 sp=0x170ac658ffb8 pc=0x22d999f\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac658ffe8 sp=0x170ac658ffe0 pc=0x1f40801\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 108\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n\ngoroutine 114 gp=0x170ac61d52c0 m=nil [select]:\nruntime.gopark(0x170ac6923f08?, 0x6?, 0x0?, 0x7?, 0x170ac6923da4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6923c08 sp=0x170ac6923be8 pc=0x1f38d2e\nruntime.selectgo(0x170ac6923f08, 0x170ac6923d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6923d38 sp=0x170ac6923c08 pc=0x1f152a5\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1500, 0x170ac699ea00, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6923f90 sp=0x170ac6923d38 pc=0x22da6a8\nnet/http.(*http2clientStream).doRequest(0x170ac61a1500, 0x0?, 0x170ac66e1f98?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6923fb8 sp=0x170ac6923f90 pc=0x22d99d8\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6923fe0 sp=0x170ac6923fb8 pc=0x22d999f\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6923fe8 sp=0x170ac6923fe0 pc=0x1f40801\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 109\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n\ngoroutine 115 gp=0x170ac61d54a0 m=nil [select]:\nruntime.gopark(0x170ac6925f08?, 0x6?, 0x38?, 0x5b?, 0x170ac6925da4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6925c08 sp=0x170ac6925be8 pc=0x1f38d2e\nruntime.selectgo(0x170ac6925f08, 0x170ac6925d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6925d38 sp=0x170ac6925c08 pc=0x1f152a5\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1680, 0x170ac699e500, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6925f90 sp=0x170ac6925d38 pc=0x22da6a8\nnet/http.(*http2clientStream).doRequest(0x170ac61a1680, 0x0?, 0x170ac66e1f98?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6925fb8 sp=0x170ac6925f90 pc=0x22d99d8\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6925fe0 sp=0x170ac6925fb8 pc=0x22d999f\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6925fe8 sp=0x170ac6925fe0 pc=0x1f40801\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 107\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n\nrax    0x0\nrbx    0x7f7eabfff640\nrcx    0x7f7ef9baa94c\nrdx    0x6\nrdi    0x10a2e\nrsi    0x10a30\nrbp    0x10a30\nrsp    0x7f7eabffe790\nr8     0x7f7eabffe860\nr9     0x7f7ef9cd04e0\nr10    0x8\nr11    0x246\nr12    0x6\nr13    0x7f7eabffeb60\nr14    0x170ac5c1f0e0\nr15    0xffffffffffffffff\nrip    0x7f7ef9baa94c\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\n\n  query-infra-both: command failed: exit status 2\noutput: runtime/cgo: pthread_create failed: Resource temporarily unavailable\nSIGABRT: abort\nPC=0x7f5e9c16994c m=2 sigcode=18446744073709551610\n\ngoroutine 0 gp=0x280029a48b40 m=2 mp=0x280029abb008 [idle]:\nruntime: g 0 gp=0x280029a48b40: unknown pc 0x7f5e9c16994c\nstack: frame={sp:0x7f5e554a7740, fp:0x0} stack=[0x7f5e54ca81e0,0x7f5e554a7de0)\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n00007f5e554a7640: 00000000 01eb64e5  00000000 00000000  .d..............\n                  ^ <internal/runtime/syscall/linux.EpollWait+0x45>\n00007f5e554a7650: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7660: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7670: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7680: 00007f5e 554a76d4  00007f5e 554a7d18  .vJU^....}JU^...\n00007f5e554a7690: 00000000 01efa894  00000000 00000000  ................\n                  ^ <runtime.netpoll+0xd4>\n00007f5e554a76a0: 00007f5e 554a76d4  00000000 00000000  .vJU^...........\n00007f5e554a76b0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a76c0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a76d0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a76e0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a76f0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7700: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7710: 00000000 00000000  00007f5e 9c176fa9  .........o..^...\n00007f5e554a7720: 00000000 00000000  00000000 00000011  ................\n00007f5e554a7730: 00000000 00000130  00007f5e 9c16993e  0.......>...^...\n00007f5e554a7740: 00000000 00000120  00000000 00000000   ...............\n                  ^ SP\n00007f5e554a7750: 00000000 00000013  00000000 00000000  ................\n00007f5e554a7760: 00000000 00000000  00007f5e 9c1eccff  ............^...\n00007f5e554a7770: 00007f5e 327fc910  00007f5e 9c1eccb1  ...2^.......^...\n00007f5e554a7780: 00007f5e 327fc640  00007f5e 554a7a50  @..2^...PzJU^...\n00007f5e554a7790: 00007f5e 554a78be  00007f5e 554a78bf  .xJU^....xJU^...\n00007f5e554a77a0: 00007f5e 327fc640  00007f5e 9c1677f5  @..2^....w..^...\n00007f5e554a77b0: 00007f5e 50000020  00000000 00000120   ..P^... .......\n00007f5e554a77c0: 00000000 003d0f00  5a23f32c fe4c8400  ..=.......L.,.#Z\n00007f5e554a77d0: 00007f5e 554a8640  00000000 00000006  @.JU^...........\n00007f5e554a77e0: 00000000 0bac42a0  00007f5e 554a7b10  .B.......{JU^...\n00007f5e554a77f0: 00002800 29a48b40  00007f5e 9c11c646  @..).(..F...^...\n00007f5e554a7800: 00007f5e 9c2d9e90  00007f5e 9c1067f3  ..-.^....g..^...\n00007f5e554a7810: 00000000 00000020  5a23f32c fe4c8400   .........L.,.#Z\n00007f5e554a7820: 00007f5e 31ffc000  00007f5e 9c1dbac4  ...1^.......^...\n00007f5e554a7830: 00000000 00000000  00000000 0000000d  ................\nruntime: g 0 gp=0x280029a48b40: unknown pc 0x7f5e9c16994c\nstack: frame={sp:0x7f5e554a7740, fp:0x0} stack=[0x7f5e54ca81e0,0x7f5e554a7de0)\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n00007f5e554a7640: 00000000 01eb64e5  00000000 00000000  .d..............\n                  ^ <internal/runtime/syscall/linux.EpollWait+0x45>\n00007f5e554a7650: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7660: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7670: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7680: 00007f5e 554a76d4  00007f5e 554a7d18  .vJU^....}JU^...\n00007f5e554a7690: 00000000 01efa894  00000000 00000000  ................\n                  ^ <runtime.netpoll+0xd4>\n00007f5e554a76a0: 00007f5e 554a76d4  00000000 00000000  .vJU^...........\n00007f5e554a76b0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a76c0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a76d0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a76e0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a76f0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7700: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7710: 00000000 00000000  00007f5e 9c176fa9  .........o..^...\n00007f5e554a7720: 00000000 00000000  00000000 00000011  ................\n00007f5e554a7730: 00000000 00000130  00007f5e 9c16993e  0.......>...^...\n00007f5e554a7740: 00000000 00000120  00000000 00000000   ...............\n                  ^ SP\n00007f5e554a7750: 00000000 00000013  00000000 00000000  ................\n00007f5e554a7760: 00000000 00000000  00007f5e 9c1eccff  ............^...\n00007f5e554a7770: 00007f5e 327fc910  00007f5e 9c1eccb1  ...2^.......^...\n00007f5e554a7780: 00007f5e 327fc640  00007f5e 554a7a50  @..2^...PzJU^...\n00007f5e554a7790: 00007f5e 554a78be  00007f5e 554a78bf  .xJU^....xJU^...\n00007f5e554a77a0: 00007f5e 327fc640  00007f5e 9c1677f5  @..2^....w..^...\n00007f5e554a77b0: 00007f5e 50000020  00000000 00000120   ..P^... .......\n00007f5e554a77c0: 00000000 003d0f00  5a23f32c fe4c8400  ..=.......L.,.#Z\n00007f5e554a77d0: 00007f5e 554a8640  00000000 00000006  @.JU^...........\n00007f5e554a77e0: 00000000 0bac42a0  00007f5e 554a7b10  .B.......{JU^...\n00007f5e554a77f0: 00002800 29a48b40  00007f5e 9c11c646  @..).(..F...^...\n00007f5e554a7800: 00007f5e 9c2d9e90  00007f5e 9c1067f3  ..-.^....g..^...\n00007f5e554a7810: 00000000 00000020  5a23f32c fe4c8400   .........L.,.#Z\n00007f5e554a7820: 00007f5e 31ffc000  00007f5e 9c1dbac4  ...1^.......^...\n00007f5e554a7830: 00000000 00000000  00000000 0000000d  ................\n\ngoroutine 1 gp=0x280029a481e0 m=nil [sync.WaitGroup.Wait]:\nruntime.gopark(0x28002a5927b8?, 0x2022a4d?, 0x30?, 0xe9?, 0x28002a5927e8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a5927a0 sp=0x28002a592780 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x280029ae34d0, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a592808 sp=0x28002a5927a0 pc=0x1f162b2\nsync.runtime_SemacquireWaitGroup(0x28002a83b578?, 0xe0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x28002a592840 sp=0x28002a592808 pc=0x1f3a66e\nsync.(*WaitGroup).Wait(0x280029ae34c8)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x28002a592868 sp=0x28002a592840 pc=0x1f4f445\ngolang.org/x/sync/errgroup.(*Group).Wait(0x280029ae34c0)\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x28002a592888 sp=0x28002a592868 pc=0x362507e\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0}, {0x28002a977b50, 0x1, 0x1})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x28002a592948 sp=0x28002a592888 pc=0x3927e05\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).gatherInfraLogs(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:587 +0x867 fp=0x28002a592d30 sp=0x28002a592948 pc=0x3927727\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:333 +0x533 fp=0x28002a593908 sp=0x28002a592d30 pc=0x3921bf3\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedInfraQueryOptions).RunInfra(0x28002a11e1e0, {0x59f0b08, 0x28002a1873b0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:83 +0x71f fp=0x28002a593b30 sp=0x28002a593908 pc=0x393887f\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawInfraQueryOptions).Run(0x0?, {0x59f0b08, 0x28002a1873b0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_options.go:109 +0x54 fp=0x28002a593b58 sp=0x28002a593b30 pc=0x3939134\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryInfraCommand.func1(0x28002a53d508?, {0x280029ca3b00?, 0x4?, 0x553b0e2?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:45 +0x28 fp=0x28002a593b80 sp=0x28002a593b58 pc=0x3938128\ngithub.com/spf13/cobra.(*Command).execute(0x28002a53d508, {0x280029ab8120, 0xc, 0xc})\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x28002a593d28 sp=0x28002a593b80 pc=0x212e614\ngithub.com/spf13/cobra.(*Command).ExecuteC(0x280029ec9b08)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x28002a593e18 sp=0x28002a593d28 pc=0x212efa5\ngithub.com/spf13/cobra.(*Command).Execute(...)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071\ngithub.com/spf13/cobra.(*Command).ExecuteContext(...)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064\nmain.main()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x28002a593f48 sp=0x28002a593e18 pc=0x47ddd6a\nruntime.main()\n\t/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x28002a593fe0 sp=0x28002a593f48 pc=0x1f01975\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a593fe8 sp=0x28002a593fe0 pc=0x1f40801\n\ngoroutine 2 gp=0x280029a48d20 m=nil [force gc (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b08fa8 sp=0x280029b08f88 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.forcegchelper()\n\t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x280029b08fe0 sp=0x280029b08fa8 pc=0x1f01c93\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b08fe8 sp=0x280029b08fe0 pc=0x1f40801\ncreated by runtime.init.7 in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\n\ngoroutine 3 gp=0x280029a492c0 m=nil [GC sweep wait]:\nruntime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b09788 sp=0x280029b09768 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.bgsweep(0x280029b30000)\n\t/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x280029b097c8 sp=0x280029b09788 pc=0x1ee9f11\nruntime.gcenable.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x280029b097e0 sp=0x280029b097c8 pc=0x1edadd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b097e8 sp=0x280029b097e0 pc=0x1f40801\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\n\ngoroutine 4 gp=0x280029a494a0 m=nil [GC scavenge wait]:\nruntime.gopark(0x10000?, 0x5981cd8?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b09f78 sp=0x280029b09f58 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.(*scavengerState).park(0x82979c0)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x280029b09fa8 sp=0x280029b09f78 pc=0x1ee77e9\nruntime.bgscavenge(0x280029b30000)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x280029b09fc8 sp=0x280029b09fa8 pc=0x1ee7d99\nruntime.gcenable.gowrap2()\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x280029b09fe0 sp=0x280029b09fc8 pc=0x1edad97\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b09fe8 sp=0x280029b09fe0 pc=0x1f40801\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\n\ngoroutine 5 gp=0x280029a49a40 m=nil [GOMAXPROCS updater (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0a788 sp=0x280029b0a768 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.updateMaxProcsGoroutine()\n\t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x280029b0a7e0 sp=0x280029b0a788 pc=0x1f10267\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0a7e8 sp=0x280029b0a7e0 pc=0x1f40801\ncreated by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\n\ngoroutine 6 gp=0x280029b481e0 m=nil [finalizer wait]:\nruntime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x280029b08601?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b08620 sp=0x280029b08600 pc=0x1f38d2e\nruntime.runFinalizers()\n\t/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x280029b087e0 sp=0x280029b08620 pc=0x1ed9cc7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b087e8 sp=0x280029b087e0 pc=0x1f40801\ncreated by runtime.createfing in goroutine 1\n\t/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d\n\ngoroutine 7 gp=0x280029d3fc20 m=nil [cleanup wait]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0af68 sp=0x280029b0af48 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.(*cleanupQueue).dequeue(0x82980e0)\n\t/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x280029b0afa0 sp=0x280029b0af68 pc=0x1ed6074\nruntime.runCleanups()\n\t/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x280029b0afe0 sp=0x280029b0afa0 pc=0x1ed66e5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0afe8 sp=0x280029b0afe0 pc=0x1f40801\ncreated by runtime.(*cleanupQueue).createGs in goroutine 1\n\t/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5\n\ngoroutine 8 gp=0x280029ee6000 m=nil [GC worker (idle)]:\nruntime.gopark(0x125c86ae32d7?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0b740 sp=0x280029b0b720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x280029ad7ab0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x280029b0b7c8 sp=0x280029b0b740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b0b7e0 sp=0x280029b0b7c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0b7e8 sp=0x280029b0b7e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 18 gp=0x280029b843c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x125c662e5759?, 0x3?, 0xc2?, 0x10?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b04740 sp=0x280029b04720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x280029ad7ab0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x280029b047c8 sp=0x280029b04740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b047e0 sp=0x280029b047c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b047e8 sp=0x280029b047e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 9 gp=0x280029ee61e0 m=nil [semacquire]:\nruntime.gopark(0x6e2b22?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029b0bc96?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0be38 sp=0x280029b0be18 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029b0bea0 sp=0x280029b0be38 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029b0bf40 sp=0x280029b0bea0 pc=0x1edbb12\nruntime.gcBgMarkWorker(0x280029ad7ab0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029b0bfc8 sp=0x280029b0bf40 pc=0x1edda88\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b0bfe0 sp=0x280029b0bfc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0bfe8 sp=0x280029b0bfe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 10 gp=0x280029ee63c0 m=nil [semacquire]:\nruntime.gopark(0x1ec6f5e?, 0x1f3ec12?, 0x80?, 0x51?, 0x1ec70df?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eec638 sp=0x280029eec618 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eec6a0 sp=0x280029eec638 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eec740 sp=0x280029eec6a0 pc=0x1edbb12\nruntime.gcBgMarkWorker(0x280029ad7ab0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eec7c8 sp=0x280029eec740 pc=0x1edda88\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eec7e0 sp=0x280029eec7c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eec7e8 sp=0x280029eec7e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 11 gp=0x280029ee65a0 m=nil [semacquire]:\nruntime.gopark(0x54b522?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029eecc96?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eece38 sp=0x280029eece18 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eecea0 sp=0x280029eece38 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eecf40 sp=0x280029eecea0 pc=0x1edbb12\nruntime.gcBgMarkWorker(0x280029ad7ab0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eecfc8 sp=0x280029eecf40 pc=0x1edda88\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eecfe0 sp=0x280029eecfc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eecfe8 sp=0x280029eecfe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 34 gp=0x280029f04000 m=nil [semacquire]:\nruntime.gopark(0x2d7122?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029ee8496?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029ee8638 sp=0x280029ee8618 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029ee86a0 sp=0x280029ee8638 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029ee8740 sp=0x280029ee86a0 pc=0x1edbb12\nruntime.gcBgMarkWorker(0x280029ad7ab0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029ee87c8 sp=0x280029ee8740 pc=0x1edda88\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029ee87e0 sp=0x280029ee87c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029ee87e8 sp=0x280029ee87e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 19 gp=0x280029b845a0 m=4 mp=0x280029b34008 [GC mark termination]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x280029b04ea0 sp=0x280029b04e90 pc=0x1f3ebc8\nruntime.forEachP(...)\n\t/usr/lib/golang/src/runtime/proc.go:2112\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x280029b04f40 sp=0x280029b04ea0 pc=0x1edbc34\nruntime.gcBgMarkWorker(0x280029ad7ab0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029b04fc8 sp=0x280029b04f40 pc=0x1edda88\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b04fe0 sp=0x280029b04fc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b04fe8 sp=0x280029b04fe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 12 gp=0x280029ee6780 m=nil [semacquire]:\nruntime.gopark(0x3f9e22?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029eed496?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eed638 sp=0x280029eed618 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eed6a0 sp=0x280029eed638 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eed740 sp=0x280029eed6a0 pc=0x1edbb12\nruntime.gcBgMarkWorker(0x280029ad7ab0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eed7c8 sp=0x280029eed740 pc=0x1edda88\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eed7e0 sp=0x280029eed7c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eed7e8 sp=0x280029eed7e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 35 gp=0x280029ee74a0 m=nil [select, locked to thread]:\nruntime.gopark(0x280029eef7a8?, 0x2?, 0x1?, 0x74?, 0x280029eef794?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eef628 sp=0x280029eef608 pc=0x1f38d2e\nruntime.selectgo(0x280029eef7a8, 0x280029eef790, 0x0?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x280029eef758 sp=0x280029eef628 pc=0x1f152a5\nruntime.ensureSigM.func1()\n\t/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x280029eef7e0 sp=0x280029eef758 pc=0x1f328e8\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eef7e8 sp=0x280029eef7e0 pc=0x1f40801\ncreated by runtime.ensureSigM in goroutine 1\n\t/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5\n\ngoroutine 50 gp=0x280029f84960 m=9 mp=0x28002a000008 [syscall]:\nruntime.notetsleepg(0x82bdea0, 0xffffffffffffffff)\n\t/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x280029ee9fa0 sp=0x280029ee9f78 pc=0x1ece509\nos/signal.signal_recv()\n\t/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x280029ee9fc0 sp=0x280029ee9fa0 pc=0x1f3b298\nos/signal.loop()\n\t/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x280029ee9fe0 sp=0x280029ee9fc0 pc=0x2030553\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029ee9fe8 sp=0x280029ee9fe0 pc=0x1f40801\ncreated by os/signal.Notify.func1.1 in goroutine 1\n\t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\n\ngoroutine 51 gp=0x280029f84b40 m=nil [select]:\nruntime.gopark(0x280029eea798?, 0x2?, 0x80?, 0xa3?, 0x280029eea78c?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eea628 sp=0x280029eea608 pc=0x1f38d2e\nruntime.selectgo(0x280029eea798, 0x280029eea788, 0x0?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x280029eea758 sp=0x280029eea628 pc=0x1f152a5\nos/signal.NotifyContext.func1()\n\t/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x280029eea7e0 sp=0x280029eea758 pc=0x202ff9c\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eea7e8 sp=0x280029eea7e0 pc=0x1f40801\ncreated by os/signal.NotifyContext in goroutine 1\n\t/usr/lib/golang/src/os/signal/signal.go:291 +0x158\n\ngoroutine 137 gp=0x28002a60a000 m=nil [sync.WaitGroup.Wait]:\nruntime.gopark(0x28002a603e00?, 0x1ed0765?, 0x80?, 0xea?, 0x7f5e54234a40?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a603da0 sp=0x28002a603d80 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x280029ae3b10, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a603e08 sp=0x28002a603da0 pc=0x1f162b2\nsync.runtime_SemacquireWaitGroup(0x28002a83b590?, 0x0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x28002a603e40 sp=0x28002a603e08 pc=0x1f3a66e\nsync.(*WaitGroup).Wait(0x280029ae3b08)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x28002a603e68 sp=0x28002a603e40 pc=0x1f4f445\ngolang.org/x/sync/errgroup.(*Group).Wait(0x280029ae3b00)\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x28002a603e88 sp=0x28002a603e68 pc=0x362507e\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x280029d80e80, {0x59f0b40, 0x280029beca00}, {0x28002a977b50, 0x1, 0x38d788c?}, 0x28002a830c40)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x28002a603f00 sp=0x28002a603e88 pc=0x392976a\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x28002a603f78 sp=0x28002a603f00 pc=0x392808e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a603fe0 sp=0x28002a603f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a603fe8 sp=0x28002a603fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 38 gp=0x28002a60b0e0 m=nil [IO wait]:\nruntime.gopark(0x28002a7c22c0?, 0x280029aee7f8?, 0x40?, 0x77?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b197b8 sp=0x280029b19798 pc=0x1f38d2e\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x280029b197f0 sp=0x280029b197b8 pc=0x1ef9e57\ninternal/poll.runtime_pollWait(0x7f5e554b6c00, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x280029b19810 sp=0x280029b197f0 pc=0x1f375c5\ninternal/poll.(*pollDesc).wait(0x280029bf6400?, 0x28002a90c000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x280029b19838 sp=0x280029b19810 pc=0x1fbd167\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0x280029bf6400, {0x28002a90c000, 0x3500, 0x3500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x280029b198d0 sp=0x280029b19838 pc=0x1fbe4ce\nnet.(*netFD).Read(0x280029bf6400, {0x28002a90c000?, 0x28002a90c000?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x280029b19918 sp=0x280029b198d0 pc=0x20798c5\nnet.(*conn).Read(0x280029a46378, {0x28002a90c000?, 0x7f5e554a9828?, 0x7f5e9c0b1e00?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x280029b19960 sp=0x280029b19918 pc=0x208b7a5\ncrypto/tls.(*atLeastReader).Read(0x28002aa26ba0, {0x28002a90c000?, 0x28002a0d8658?, 0x1efcc28?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x280029b199a8 sp=0x280029b19960 pc=0x2222ddb\nbytes.(*Buffer).ReadFrom(0x28002a0d8628, {0x59a3680, 0x28002aa26ba0})\n\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x280029b19a00 sp=0x280029b199a8 pc=0x1fe9258\ncrypto/tls.(*Conn).readFromUntil(0x28002a0d8388, {0x59a32c0, 0x280029a46378}, 0x222764c?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x280029b19a38 sp=0x280029b19a00 pc=0x2222fbe\ncrypto/tls.(*Conn).readRecordOrCCS(0x28002a0d8388, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x280029b19c18 sp=0x280029b19a38 pc=0x222009b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:588\ncrypto/tls.(*Conn).Read(0x28002a0d8388, {0x280029fed000, 0x1000, 0x22c34e0?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x280029b19c80 sp=0x280029b19c18 pc=0x22267c5\nbufio.(*Reader).Read(0x28002a7b89c0, {0x28002a7c8584, 0x9, 0x22e100e?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0x280029b19cb8 sp=0x280029b19c80 pc=0x20a71f7\nio.ReadAtLeast({0x59a1260, 0x28002a7b89c0}, {0x28002a7c8584, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0x280029b19d00 sp=0x280029b19cb8 pc=0x1fb17ee\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\nnet/http.http2readFrameHeader({0x28002a7c8584, 0x9, 0x280000000015?}, {0x59a1260?, 0x28002a7b89c0?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:1805 +0x65 fp=0x280029b19d50 sp=0x280029b19d00 pc=0x22b9e25\nnet/http.(*http2Framer).ReadFrameHeader(0x28002a7c8540)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:2071 +0x6b fp=0x280029b19dd8 sp=0x280029b19d50 pc=0x22ba9eb\nnet/http.(*http2Framer).ReadFrame(0x28002a7c8540)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:2130 +0x18 fp=0x280029b19e18 sp=0x280029b19dd8 pc=0x22bb1b8\nnet/http.(*http2clientConnReadLoop).run(0x280029b19fa8)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9550 +0xca fp=0x280029b19f60 sp=0x280029b19e18 pc=0x22de14a\nnet/http.(*http2ClientConn).readLoop(0x280029bc01c0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x280029b19fc8 sp=0x280029b19f60 pc=0x22dd532\nnet/http.(*http2Transport).newClientConn.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x280029b19fe0 sp=0x280029b19fc8 pc=0x22d7057\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b19fe8 sp=0x280029b19fe0 pc=0x1f40801\ncreated by net/http.(*http2Transport).newClientConn in goroutine 37\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5\n\ngoroutine 69 gp=0x28002a60b4a0 m=nil [IO wait]:\nruntime.gopark(0x28002a50c560?, 0x1edb176?, 0x80?, 0xed?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a606808 sp=0x28002a6067e8 pc=0x1f38d2e\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x28002a606840 sp=0x28002a606808 pc=0x1ef9e57\ninternal/poll.runtime_pollWait(0x7f5e554b6a00, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x28002a606860 sp=0x28002a606840 pc=0x1f375c5\ninternal/poll.(*pollDesc).wait(0x280029bf6c80?, 0x280029a5a000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x28002a606888 sp=0x28002a606860 pc=0x1fbd167\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0x280029bf6c80, {0x280029a5a000, 0x2500, 0x2500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x28002a606920 sp=0x28002a606888 pc=0x1fbe4ce\nnet.(*netFD).Read(0x280029bf6c80, {0x280029a5a000?, 0x28002a60690a?, 0x1edb176?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x28002a606968 sp=0x28002a606920 pc=0x20798c5\nnet.(*conn).Read(0x280029a463c0, {0x280029a5a000?, 0x7f5e5423eae0?, 0x7f5e9c09dde0?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x28002a6069b0 sp=0x28002a606968 pc=0x208b7a5\ncrypto/tls.(*atLeastReader).Read(0x28002a628318, {0x280029a5a000?, 0x28002a606a98?, 0x1efcc28?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x28002a6069f8 sp=0x28002a6069b0 pc=0x2222ddb\nbytes.(*Buffer).ReadFrom(0x280029ba9ea8, {0x59a3680, 0x28002a628318})\n\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x28002a606a50 sp=0x28002a6069f8 pc=0x1fe9258\ncrypto/tls.(*Conn).readFromUntil(0x280029ba9c08, {0x59a32c0, 0x280029a463c0}, 0x222764c?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x28002a606a88 sp=0x28002a606a50 pc=0x2222fbe\ncrypto/tls.(*Conn).readRecordOrCCS(0x280029ba9c08, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x28002a606c68 sp=0x28002a606a88 pc=0x222009b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:588\ncrypto/tls.(*Conn).Read(0x280029ba9c08, {0x28002a24a000, 0x1000, 0x4e55540?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x28002a606cd0 sp=0x28002a606c68 pc=0x22267c5\nnet/http.(*persistConn).Read(0x28002a0ad400, {0x28002a24a000?, 0x59a0ea0?, 0x81260e0?})\n\t/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x28002a606d30 sp=0x28002a606cd0 pc=0x2319d07\nbufio.(*Reader).fill(0x28002a8130e0)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x28002a606d68 sp=0x28002a606d30 pc=0x20a6c63\nbufio.(*Reader).Peek(0x28002a8130e0, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x28002a606d80 sp=0x28002a606d68 pc=0x20a6df3\nnet/http.(*persistConn).readLoop(0x28002a0ad400)\n\t/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x28002a606fc8 sp=0x28002a606d80 pc=0x231a812\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x28002a606fe0 sp=0x28002a606fc8 pc=0x2319197\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a606fe8 sp=0x28002a606fe0 pc=0x1f40801\ncreated by net/http.(*Transport).dialConn in goroutine 55\n\t/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac\n\ngoroutine 70 gp=0x28002a60bc20 m=nil [select]:\nruntime.gopark(0x28002a613f48?, 0x2?, 0x0?, 0xb6?, 0x28002a613ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a708d88 sp=0x28002a708d68 pc=0x1f38d2e\nruntime.selectgo(0x28002a708f48, 0x28002a613ef0, 0x280029c93a80?, 0x0, 0x28002a4f7c80?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a708eb8 sp=0x28002a708d88 pc=0x1f152a5\nnet/http.(*persistConn).writeLoop(0x28002a0ad400)\n\t/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x28002a708fc8 sp=0x28002a708eb8 pc=0x231c426\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x28002a708fe0 sp=0x28002a708fc8 pc=0x2319157\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a708fe8 sp=0x28002a708fe0 pc=0x1f40801\ncreated by net/http.(*Transport).dialConn in goroutine 55\n\t/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe\n\ngoroutine 138 gp=0x28002a84a780 m=nil [semacquire]:\nruntime.gopark(0x1?, 0x1f78fa5?, 0xe0?, 0x75?, 0x2?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a2ca910 sp=0x28002a2ca8f0 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a2ca978 sp=0x28002a2ca910 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x28002a2caa18 sp=0x28002a2ca978 pc=0x1edbb12\nruntime.gcAssistAlloc(0x28002a84a780)\n\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x28002a2caaa8 sp=0x28002a2caa18 pc=0x1ee06b6\nruntime.deductAssistCredit(0x28002a2cac70?)\n\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x28002a2caac8 sp=0x28002a2caaa8 pc=0x1ed10a6\nruntime.mallocgc(0x10, 0x4af39e0, 0x1)\n\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x28002a2caaf8 sp=0x28002a2caac8 pc=0x1f35e07\nruntime.convTstring({0x28002a6a7728, 0x14})\n\t/usr/lib/golang/src/runtime/iface.go:431 +0x45 fp=0x28002a2cab20 sp=0x28002a2caaf8 pc=0x1f35865\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x280029beca00}, 0x28002a830cb0, 0x280029f636e0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:297 +0x394 fp=0x28002a2caf48 sp=0x28002a2cab20 pc=0x3920674\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x28002a2caf78 sp=0x28002a2caf48 pc=0x392800c\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a2cafe0 sp=0x28002a2caf78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a2cafe8 sp=0x28002a2cafe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 139 gp=0x28002a84a960 m=nil [select]:\nruntime.gopark(0x28002a4bdd00?, 0x2?, 0x1?, 0xc9?, 0x28002a4bdc74?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a4bdac8 sp=0x28002a4bdaa8 pc=0x1f38d2e\nruntime.selectgo(0x28002a4bdd00, 0x28002a4bdc70, 0x28002a3e8a50?, 0x0, 0x280029f827c8?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a4bdbf8 sp=0x28002a4bdac8 pc=0x1f152a5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x49424d554554534c?, {0x59f0b40, 0x280029beca00}, 0x28002a830c40, 0x28002a830cb0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x28002a4bdf10 sp=0x28002a4bdbf8 pc=0x392836e\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x28002a4bdf78 sp=0x28002a4bdf10 pc=0x3927f3d\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a4bdfe0 sp=0x28002a4bdf78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a4bdfe8 sp=0x28002a4bdfe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 140 gp=0x28002a84ab40 m=nil [semacquire]:\nruntime.gopark(0x1f6aa32?, 0x28002a34b418?, 0xc5?, 0xbd?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a34b3c0 sp=0x28002a34b3a0 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a34b428 sp=0x28002a34b3c0 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x28002a34b4c8 sp=0x28002a34b428 pc=0x1edbb12\nruntime.gcAssistAlloc(0x28002a84ab40)\n\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x28002a34b558 sp=0x28002a34b4c8 pc=0x1ee06b6\nruntime.deductAssistCredit(0x20?)\n\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x28002a34b578 sp=0x28002a34b558 pc=0x1ed10a6\nruntime.mallocgc(0x1000, 0x4af32a0, 0x1)\n\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x28002a34b5a8 sp=0x28002a34b578 pc=0x1f35e07\nruntime.makeslice(0x28002a34b668?, 0x1f8?, 0x1ff?)\n\t/usr/lib/golang/src/runtime/slice.go:117 +0x49 fp=0x28002a34b5d0 sp=0x28002a34b5a8 pc=0x1f3b489\nbufio.NewWriterSize(...)\n\t/usr/lib/golang/src/bufio/bufio.go:599\nbufio.NewWriter(...)\n\t/usr/lib/golang/src/bufio/bufio.go:608\nencoding/csv.NewWriter(...)\n\t/usr/lib/golang/src/encoding/csv/writer.go:42\ngithub.com/Azure/azure-kusto-go/azkustodata/query.(*row).String(0x812ab90?)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/row_impl.go:100 +0x18e fp=0x28002a34b6e8 sp=0x28002a34b5d0 pc=0x38cf5ee\nfmt.(*pp).handleMethods(0x28002a24c0d0, 0x2a34b8f0?)\n\t/usr/lib/golang/src/fmt/print.go:674 +0x2f4 fp=0x28002a34b8a8 sp=0x28002a34b6e8 pc=0x1fdd1d4\nfmt.(*pp).printArg(0x28002a24c0d0, {0x549a3a0, 0x28002a7c7c80}, 0x76)\n\t/usr/lib/golang/src/fmt/print.go:757 +0x42e fp=0x28002a34b938 sp=0x28002a34b8a8 pc=0x1fdd9ee\nfmt.(*pp).doPrintf(0x28002a24c0d0, {0x553a400, 0x2}, {0x28002a34bd08, 0x1, 0x1})\n\t/usr/lib/golang/src/fmt/print.go:1075 +0x3ec fp=0x28002a34ba20 sp=0x28002a34b938 pc=0x1fe04ec\nfmt.Sprintf({0x553a400, 0x2}, {0x28002a34bd08, 0x1, 0x1})\n\t/usr/lib/golang/src/fmt/print.go:239 +0x53 fp=0x28002a34ba78 sp=0x28002a34ba20 pc=0x1fda6b3\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x280029e454e0, {0x59f0b40, 0x280029beca50}, {0x5a00300, 0x280029ae2040}, 0x28002a830c40)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:160 +0x8d8 fp=0x28002a34bd60 sp=0x28002a34ba78 pc=0x391bad8\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x2d20414320746f6f?, {0x59f0b40?, 0x280029beca50?}, {0x5a00300?, 0x280029ae2040?}, 0x43437a4b4349494d?)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x28002a34be88 sp=0x28002a34bd60 pc=0x3929c16\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x28002a34bf78 sp=0x28002a34be88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a34bfe0 sp=0x28002a34bf78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a34bfe8 sp=0x28002a34bfe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 137\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 142 gp=0x28002a84af00 m=nil [select]:\nruntime.gopark(0x28002a333de0?, 0x2?, 0x8?, 0xb8?, 0x28002a333dbc?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a333c58 sp=0x28002a333c38 pc=0x1f38d2e\nruntime.selectgo(0x28002a333de0, 0x28002a333db8, 0x280029afc008?, 0x1, 0x1f3665f?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a333d88 sp=0x28002a333c58 pc=0x1f152a5\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.(*iterativeDataset).sendTable(0x280029becf00, {0x5a19be0, 0x28002ab876c0})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:378 +0xf3 fp=0x28002a333e10 sp=0x28002a333d88 pc=0x38d8d33\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.handleDataTable(0x280029becf00, 0x28002a3b2b40)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:324 +0x2c6 fp=0x28002a333ec8 sp=0x28002a333e10 pc=0x38d8686\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x280029becf00)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:167 +0x148 fp=0x28002a333f30 sp=0x28002a333ec8 pc=0x38d7b48\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x280029becf00, 0x28002a977da0)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x28002a333fc0 sp=0x28002a333f30 pc=0x38d788c\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x28002a333fe0 sp=0x28002a333fc0 pc=0x38d751b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a333fe8 sp=0x28002a333fe0 pc=0x1f40801\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 140\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc\n\nrax    0x0\nrbx    0x7f5e554a8640\nrcx    0x7f5e9c16994c\nrdx    0x6\nrdi    0x10a66\nrsi    0x10a67\nrbp    0x10a67\nrsp    0x7f5e554a7740\nr8     0x7f5e554a7810\nr9     0x7f5e9c28f4e0\nr10    0x8\nr11    0x246\nr12    0x6\nr13    0x7f5e554a7b10\nr14    0x280029a48b40\nr15    0xffffffffffffffff\nrip    0x7f5e9c16994c\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\n",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:123]: Timed out after 651.749s.
Expected success, but got an error:
    <*errors.errorString | 0x48c3b8dec90>: 
    must-gather CLI test failures:
      query-basic: command failed: exit status 1
    output: �[37m[21:33:14.493]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-basic-3888395104"
    }�[0m
    �[37m[21:33:23.438]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:33:21.8572874Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\", ActivityId:\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\", SubActivityId:\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\"}, IsPermanent:false})",
      "name": "detailedInfraOrchestrationLogs"
    }�[0m
    �[37m[21:33:23.438]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "query \"backendControllerConditions\" time window 50/289 failed (KQL: let _startTime = datetime(2026-10-15T01:38:14.4931102Z);\nlet _endTime = datetime(2026-10-15T01:43:14.4931101Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-15T01:38:14.4931102Z);\nlet _endTime = datetime(2026-10-15T01:43:14.4931101Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context canceled",
      "name": "backendControllerConditions"
    }�[0m
    �[37m[21:33:23.712]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{
      "input": "/tmp/must-gather-cli-e2e-query-basic-3888395104/custom/custom-query_cosmosResourceSnapshots.jsonl",
      "output": "/tmp/must-gather-cli-e2e-query-basic-3888395104/cosmosContent"
    }�[0m
    �[37m[21:33:23.712]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-basic-3888395104/custom/custom-query_cosmosResourceSnapshots.jsonl"
    }�[0m
    �[37m[21:33:24.208]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{
      "count": 1498,
      "files": 1
    }�[0m
    �[37m[21:33:24.208]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-basic-3888395104/cosmosContent"
    }�[0m
    �[37m[21:33:24.217]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m
    �[37m[21:33:40.606]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{
      "commits": 793
    }�[0m
    �[37m[21:33:40.607]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{
      "clusterIDs": null,
      "output": "/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect"
    }�[0m
    �[37m[21:33:40.961]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{
      "clusterIDs": [
        "2tfevok49t0kjg6c48lr24m7jgn4lk62"
      ]
    }�[0m
    �[37m[21:33:41.088]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{
      "clusterNames": [
        "ci01-j8700672-mgmt-1",
        "ci01-j8700672-svc"
      ]
    }�[0m
    �[37m[21:33:41.127]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespaces": [
        "2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g"
      ]
    }�[0m
    �[37m[21:33:41.127]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{
      "managementCluster": "ci01-j8700672-mgmt-1",
      "namespaceCount": 4
    }�[0m
    �[37m[21:33:41.127]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:33:41.317]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:33:41.478]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:33:41.623]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g",
      "output": "/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g"
    }�[0m
    �[37m[21:33:43.044]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:33:21.8572874Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\", ActivityId:\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\", SubActivityId:\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\"}, IsPermanent:false})",
      "output": "/tmp/must-gather-cli-e2e-query-basic-3888395104"
    }�[0m
    �[37m[21:33:43.044]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:33:21.8572874Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\", ActivityId:\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\", SubActivityId:\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\"}, IsPermanent:false})"
    }�[0m
    
      query-skip-hcp-logs: command failed: exit status 1
    output: �[37m[21:33:43.407]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843"
    }�[0m
    �[37m[21:33:49.648]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:33:47.6716888Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\", ActivityId:\"4f13c479-3f1c-438a-b976-532417bcd103\", SubActivityId:\"4f13c479-3f1c-438a-b976-532417bcd103\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"4f13c479-3f1c-438a-b976-532417bcd103\", ActivityStack:\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\"}, IsPermanent:false})",
      "name": "detailedInfraOrchestrationLogs"
    }�[0m
    �[37m[21:33:49.648]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "query \"backendControllerConditions\" time window 47/289 failed (KQL: let _startTime = datetime(2026-10-15T01:23:43.4070857Z);\nlet _endTime = datetime(2026-10-15T01:28:43.4070856Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled",
      "name": "backendControllerConditions"
    }�[0m
    �[37m[21:33:49.744]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{
      "input": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/custom/custom-query_cosmosResourceSnapshots.jsonl",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/cosmosContent"
    }�[0m
    �[37m[21:33:49.744]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/custom/custom-query_cosmosResourceSnapshots.jsonl"
    }�[0m
    �[37m[21:33:50.271]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{
      "count": 1498,
      "files": 1
    }�[0m
    �[37m[21:33:50.271]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/cosmosContent"
    }�[0m
    �[37m[21:33:50.281]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m
    �[37m[21:34:08.647]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{
      "commits": 793
    }�[0m
    �[37m[21:34:08.648]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{
      "clusterIDs": null,
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect"
    }�[0m
    �[37m[21:34:09.212]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{
      "clusterIDs": [
        "2tfevok49t0kjg6c48lr24m7jgn4lk62"
      ]
    }�[0m
    �[37m[21:34:09.309]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{
      "clusterNames": [
        "ci01-j8700672-mgmt-1",
        "ci01-j8700672-svc"
      ]
    }�[0m
    �[37m[21:34:09.470]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespaces": [
        "2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g"
      ]
    }�[0m
    �[37m[21:34:09.470]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{
      "managementCluster": "ci01-j8700672-mgmt-1",
      "namespaceCount": 4
    }�[0m
    �[37m[21:34:09.470]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:34:09.615]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:34:10.183]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:34:10.617]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g"
    }�[0m
    �[37m[21:34:12.156]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:33:47.6716888Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\", ActivityId:\"4f13c479-3f1c-438a-b976-532417bcd103\", SubActivityId:\"4f13c479-3f1c-438a-b976-532417bcd103\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"4f13c479-3f1c-438a-b976-532417bcd103\", ActivityStack:\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\"}, IsPermanent:false})",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843"
    }�[0m
    �[37m[21:34:12.157]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:33:47.6716888Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\", ActivityId:\"4f13c479-3f1c-438a-b976-532417bcd103\", SubActivityId:\"4f13c479-3f1c-438a-b976-532417bcd103\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"4f13c479-3f1c-438a-b976-532417bcd103\", ActivityStack:\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\"}, IsPermanent:false})"
    }�[0m
    
      query-skip-kubernetes-events: command failed: exit status 1
    output: �[37m[21:34:12.559]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767"
    }�[0m
    �[37m[21:34:18.220]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:34:16.3210435Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\", ActivityId:\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\", SubActivityId:\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\"}, IsPermanent:false})",
      "name": "detailedInfraOrchestrationLogs"
    }�[0m
    �[37m[21:34:18.221]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "query \"backendControllerConditions\" time window 47/289 failed (KQL: let _startTime = datetime(2026-10-15T01:24:12.5582990Z);\nlet _endTime = datetime(2026-10-15T01:29:12.5582989Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled",
      "name": "backendControllerConditions"
    }�[0m
    �[37m[21:34:18.221]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{
      "input": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/custom/custom-query_cosmosResourceSnapshots.jsonl",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/cosmosContent"
    }�[0m
    �[37m[21:34:18.221]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/custom/custom-query_cosmosResourceSnapshots.jsonl"
    }�[0m
    �[37m[21:34:18.723]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{
      "count": 1498,
      "files": 1
    }�[0m
    �[37m[21:34:18.724]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/cosmosContent"
    }�[0m
    �[37m[21:34:18.734]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m
    �[37m[21:34:35.437]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{
      "clusterIDs": null,
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect"
    }�[0m
    �[37m[21:34:35.845]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{
      "clusterIDs": [
        "2tfevok49t0kjg6c48lr24m7jgn4lk62"
      ]
    }�[0m
    �[37m[21:34:36.194]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{
      "clusterNames": [
        "ci01-j8700672-mgmt-1",
        "ci01-j8700672-svc"
      ]
    }�[0m
    �[37m[21:34:36.313]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespaces": [
        "2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g"
      ]
    }�[0m
    �[37m[21:34:36.313]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{
      "managementCluster": "ci01-j8700672-mgmt-1",
      "namespaceCount": 4
    }�[0m
    �[37m[21:34:36.313]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:34:36.544]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:34:36.863]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:34:37.093]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g"
    }�[0m
    �[37m[21:34:38.438]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:34:16.3210435Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\", ActivityId:\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\", SubActivityId:\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\"}, IsPermanent:false})\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\n",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767"
    }�[0m
    �[37m[21:34:38.438]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:34:16.3210435Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\", ActivityId:\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\", SubActivityId:\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\"}, IsPermanent:false})\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\n"
    }�[0m
    
      query-collect-systemd-logs: command failed: exit status 1
    output: �[37m[21:34:38.772]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904"
    }�[0m
    �[37m[21:34:46.539]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:34:46.3046137Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\", ActivityId:\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\", SubActivityId:\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\", ActivityStack:\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\"}, IsPermanent:false})",
      "name": "detailedInfraOrchestrationLogs"
    }�[0m
    �[37m[21:34:46.539]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "query \"backendControllerConditions\" time window 45/289 failed (KQL: let _startTime = datetime(2026-10-15T01:14:38.7716298Z);\nlet _endTime = datetime(2026-10-15T01:19:38.7716297Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled",
      "name": "backendControllerConditions"
    }�[0m
    �[37m[21:34:47.129]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{
      "input": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/custom/custom-query_cosmosResourceSnapshots.jsonl",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/cosmosContent"
    }�[0m
    �[37m[21:34:47.129]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/custom/custom-query_cosmosResourceSnapshots.jsonl"
    }�[0m
    �[37m[21:34:47.633]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{
      "count": 1498,
      "files": 1
    }�[0m
    �[37m[21:34:47.633]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/cosmosContent"
    }�[0m
    �[37m[21:34:47.642]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m
    �[37m[21:34:59.291]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{
      "clusterIDs": null,
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect"
    }�[0m
    �[37m[21:34:59.911]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{
      "clusterIDs": [
        "2tfevok49t0kjg6c48lr24m7jgn4lk62"
      ]
    }�[0m
    �[37m[21:35:00.221]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{
      "clusterNames": [
        "ci01-j8700672-mgmt-1",
        "ci01-j8700672-svc"
      ]
    }�[0m
    �[37m[21:35:00.360]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespaces": [
        "2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62",
        "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g"
      ]
    }�[0m
    �[37m[21:35:00.360]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{
      "managementCluster": "ci01-j8700672-mgmt-1",
      "namespaceCount": 4
    }�[0m
    �[37m[21:35:00.360]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:35:00.702]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:35:01.111]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62"
    }�[0m
    �[37m[21:35:01.638]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j8700672-mgmt-1",
      "namespace": "ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g"
    }�[0m
    �[37m[21:35:02.339]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:34:46.3046137Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\", ActivityId:\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\", SubActivityId:\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\", ActivityStack:\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\"}, IsPermanent:false})\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\n",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904"
    }�[0m
    �[37m[21:35:02.340]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:34:46.3046137Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\", ActivityId:\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\", SubActivityId:\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\", ActivityStack:\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\"}, IsPermanent:false})\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\n"
    }�[0m
    
      query-skip-hcp-and-events: command failed: exit status 2
    output: �[37m[21:35:02.599]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-1950984811"
    }�[0m
    runtime/cgo: pthread_create failed: Resource temporarily unavailable
    SIGABRT: abort
    PC=0x7f80df60594c m=4 sigcode=18446744073709551610
    
    goroutine 0 gp=0x14197f1334a0 m=4 mp=0x14197f21a008 [idle]:
    runtime: g 0 gp=0x14197f1334a0: unknown pc 0x7f80df60594c
    stack: frame={sp:0x7f808bffe790, fp:0x0} stack=[0x7f808b7ff1e0,0x7f808bffede0)
                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef
    00007f808bffe690: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6a0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6b0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6c0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6d0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6e0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6f0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe700: 00000000 00000000  00000000 00000000  ................
    00007f808bffe710: 00000000 00000000  00000000 00000000  ................
    00007f808bffe720: 00000000 00000000  00000000 00000000  ................
    00007f808bffe730: 00000000 00000000  00000000 00000000  ................
    00007f808bffe740: 00000000 00000000  00000000 00000000  ................
    00007f808bffe750: 00000000 00000000  00000000 00000000  ................
    00007f808bffe760: 00000000 00000000  00007f80 df612fa9  ........./a.....
    00007f808bffe770: 00000000 00000000  00000000 00000011  ................
    00007f808bffe780: 00000000 00000130  00007f80 df60593e  0.......>Y`.....
    00007f808bffe790: 00000000 00000120  00000000 00000000   ...............
                      ^ SP
    00007f808bffe7a0: 00010000 00000013  00000000 00000000  ................
    00007f808bffe7b0: 00000000 00000000  00007f80 df688cff  ..........h.....
    00007f808bffe7c0: 00007f80 89ffb910  00007f80 df688cb1  ..........h.....
    00007f808bffe7d0: 00007f80 89ffb640  00007f80 8bffeaa0  @...............
    00007f808bffe7e0: 00007f80 8bffe90e  00007f80 8bffe90f  ................
    00007f808bffe7f0: 00007f80 89ffb640  00007f80 df6037f5  @........7`.....
    00007f808bffe800: 00007f80 84000020  00000000 00000120   ....... .......
    00007f808bffe810: 00000000 003d0f00  701e1ebb c8982900  ..=......).....p
    00007f808bffe820: 00007f80 8bfff640  00000000 00000006  @...............
    00007f808bffe830: 00007f80 84000c90  00007f80 8bffeb60  ........`.......
    00007f808bffe840: 00001419 7f1334a0  00007f80 df5b8646  .4......F.[.....
    00007f808bffe850: 00007f80 df775e90  00007f80 df5a27f3  .^w......'Z.....
    00007f808bffe860: 00000000 00000020  701e1ebb c8982900   ........).....p
    00007f808bffe870: 00007f80 897fb000  00007f80 df677ac4  .........zg.....
    00007f808bffe880: 00000000 00000000  00000000 0000000d  ................
    runtime: g 0 gp=0x14197f1334a0: unknown pc 0x7f80df60594c
    stack: frame={sp:0x7f808bffe790, fp:0x0} stack=[0x7f808b7ff1e0,0x7f808bffede0)
                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef
    00007f808bffe690: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6a0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6b0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6c0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6d0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6e0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe6f0: 00000000 00000000  00000000 00000000  ................
    00007f808bffe700: 00000000 00000000  00000000 00000000  ................
    00007f808bffe710: 00000000 00000000  00000000 00000000  ................
    00007f808bffe720: 00000000 00000000  00000000 00000000  ................
    00007f808bffe730: 00000000 00000000  00000000 00000000  ................
    00007f808bffe740: 00000000 00000000  00000000 00000000  ................
    00007f808bffe750: 00000000 00000000  00000000 00000000  ................
    00007f808bffe760: 00000000 00000000  00007f80 df612fa9  ........./a.....
    00007f808bffe770: 00000000 00000000  00000000 00000011  ................
    00007f808bffe780: 00000000 00000130  00007f80 df60593e  0.......>Y`.....
    00007f808bffe790: 00000000 00000120  00000000 00000000   ...............
                      ^ SP
    00007f808bffe7a0: 00010000 00000013  00000000 00000000  ................
    00007f808bffe7b0: 00000000 00000000  00007f80 df688cff  ..........h.....
    00007f808bffe7c0: 00007f80 89ffb910  00007f80 df688cb1  ..........h.....
    00007f808bffe7d0: 00007f80 89ffb640  00007f80 8bffeaa0  @...............
    00007f808bffe7e0: 00007f80 8bffe90e  00007f80 8bffe90f  ................
    00007f808bffe7f0: 00007f80 89ffb640  00007f80 df6037f5  @........7`.....
    00007f808bffe800: 00007f80 84000020  00000000 00000120   ....... .......
    00007f808bffe810: 00000000 003d0f00  701e1ebb c8982900  ..=......).....p
    00007f808bffe820: 00007f80 8bfff640  00000000 00000006  @...............
    00007f808bffe830: 00007f80 84000c90  00007f80 8bffeb60  ........`.......
    00007f808bffe840: 00001419 7f1334a0  00007f80 df5b8646  .4......F.[.....
    00007f808bffe850: 00007f80 df775e90  00007f80 df5a27f3  .^w......'Z.....
    00007f808bffe860: 00000000 00000020  701e1ebb c8982900   ........).....p
    00007f808bffe870: 00007f80 897fb000  00007f80 df677ac4  .........zg.....
    00007f808bffe880: 00000000 00000000  00000000 0000000d  ................
    
    goroutine 1 gp=0x14197f1321e0 m=nil [sync.WaitGroup.Wait]:
    runtime.gopark(0x14197fe86c00?, 0x2022a4d?, 0x10?, 0x63?, 0x14197fe86c30?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197ff32be8 sp=0x14197ff32bc8 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x14197fc98610, 0x0, 0x1, 0x0, 0x19)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197ff32c50 sp=0x14197ff32be8 pc=0x1f162b2
    sync.runtime_SemacquireWaitGroup(0x14197fd289c0?, 0xe0?)
    	/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x14197ff32c88 sp=0x14197ff32c50 pc=0x1f3a66e
    sync.(*WaitGroup).Wait(0x14197fc98608)
    	/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x14197ff32cb0 sp=0x14197ff32c88 pc=0x1f4f445
    golang.org/x/sync/errgroup.(*Group).Wait(0x14197fc98600)
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x14197ff32cd0 sp=0x14197ff32cb0 pc=0x362507e
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x14197f521dc0, {0x59f0b08, 0x14197f530fc0}, {0x14197fcca000, 0x15, 0x1c})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x14197ff32d90 sp=0x14197ff32cd0 pc=0x3927e05
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x14197f521dc0, {0x59f0b08, 0x14197f530fc0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:500 +0x4b08 fp=0x14197ff33968 sp=0x14197ff32d90 pc=0x39261c8
    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedQueryOptions).RunQuery(0x14197f1401e0, {0x59f0b08, 0x14197f530fc0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_cmd.go:75 +0x565 fp=0x14197ff33b30 sp=0x14197ff33968 pc=0x3937005
    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawQueryOptions).Run(0x0?, {0x59f0b08, 0x14197f530fc0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_options.go:178 +0x54 fp=0x14197ff33b58 sp=0x14197ff33b30 pc=0x3939d34
    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryCommand.func1(0x14197f4ba008?, {0x14197f5f82a0?, 0x4?, 0x553b0e2?})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_cmd.go:45 +0x28 fp=0x14197ff33b80 sp=0x14197ff33b58 pc=0x3936a68
    github.com/spf13/cobra.(*Command).execute(0x14197f4ba008, {0x14197f2c0030, 0xe, 0xf})
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x14197ff33d28 sp=0x14197ff33b80 pc=0x212e614
    github.com/spf13/cobra.(*Command).ExecuteC(0x14197f625b08)
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x14197ff33e18 sp=0x14197ff33d28 pc=0x212efa5
    github.com/spf13/cobra.(*Command).Execute(...)
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071
    github.com/spf13/cobra.(*Command).ExecuteContext(...)
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064
    main.main()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x14197ff33f48 sp=0x14197ff33e18 pc=0x47ddd6a
    runtime.main()
    	/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x14197ff33fe0 sp=0x14197ff33f48 pc=0x1f01975
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197ff33fe8 sp=0x14197ff33fe0 pc=0x1f40801
    
    goroutine 2 gp=0x14197f132d20 m=nil [force gc (idle)]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f2fa8 sp=0x14197f1f2f88 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.forcegchelper()
    	/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x14197f1f2fe0 sp=0x14197f1f2fa8 pc=0x1f01c93
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f2fe8 sp=0x14197f1f2fe0 pc=0x1f40801
    created by runtime.init.7 in goroutine 1
    	/usr/lib/golang/src/runtime/proc.go:363 +0x1a
    
    goroutine 18 gp=0x14197f2843c0 m=nil [GC sweep wait]:
    runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1ee788 sp=0x14197f1ee768 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.bgsweep(0x14197f292000)
    	/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x14197f1ee7c8 sp=0x14197f1ee788 pc=0x1ee9f11
    runtime.gcenable.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x14197f1ee7e0 sp=0x14197f1ee7c8 pc=0x1edadd7
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1ee7e8 sp=0x14197f1ee7e0 pc=0x1f40801
    created by runtime.gcenable in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:214 +0x66
    
    goroutine 19 gp=0x14197f2845a0 m=nil [GC scavenge wait]:
    runtime.gopark(0x138d6d?, 0x3979fd?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1eef78 sp=0x14197f1eef58 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.(*scavengerState).park(0x82979c0)
    	/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x14197f1eefa8 sp=0x14197f1eef78 pc=0x1ee77e9
    runtime.bgscavenge(0x14197f292000)
    	/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x14197f1eefc8 sp=0x14197f1eefa8 pc=0x1ee7d99
    runtime.gcenable.gowrap2()
    	/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x14197f1eefe0 sp=0x14197f1eefc8 pc=0x1edad97
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1eefe8 sp=0x14197f1eefe0 pc=0x1f40801
    created by runtime.gcenable in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:215 +0xa5
    
    goroutine 20 gp=0x14197f284780 m=nil [GOMAXPROCS updater (idle)]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1ef788 sp=0x14197f1ef768 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.updateMaxProcsGoroutine()
    	/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x14197f1ef7e0 sp=0x14197f1ef788 pc=0x1f10267
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1ef7e8 sp=0x14197f1ef7e0 pc=0x1f40801
    created by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1
    	/usr/lib/golang/src/runtime/proc.go:7083 +0x37
    
    goroutine 21 gp=0x14197f284d20 m=nil [finalizer wait]:
    runtime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x14197f1f2601?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f2620 sp=0x14197f1f2600 pc=0x1f38d2e
    runtime.runFinalizers()
    	/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x14197f1f27e0 sp=0x14197f1f2620 pc=0x1ed9cc7
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f27e8 sp=0x14197f1f27e0 pc=0x1f40801
    created by runtime.createfing in goroutine 1
    	/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d
    
    goroutine 22 gp=0x14197f47c960 m=nil [cleanup wait]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1eff68 sp=0x14197f1eff48 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.(*cleanupQueue).dequeue(0x82980e0)
    	/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x14197f1effa0 sp=0x14197f1eff68 pc=0x1ed6074
    runtime.runCleanups()
    	/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x14197f1effe0 sp=0x14197f1effa0 pc=0x1ed66e5
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1effe8 sp=0x14197f1effe0 pc=0x1f40801
    created by runtime.(*cleanupQueue).createGs in goroutine 1
    	/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5
    
    goroutine 23 gp=0x14197f5a30e0 m=nil [GC worker (idle)]:
    runtime.gopark(0x125b862b4ec7?, 0x1?, 0x21?, 0xab?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f0740 sp=0x14197f1f0720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x14197f29d9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f07c8 sp=0x14197f1f0740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f07e0 sp=0x14197f1f07c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f07e8 sp=0x14197f1f07e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 3 gp=0x14197f133680 m=nil [runnable]:
    runtime.gopark(0x125b8625d223?, 0x3?, 0x89?, 0x19?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f3740 sp=0x14197f1f3720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x14197f29d9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f37c8 sp=0x14197f1f3740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f37e0 sp=0x14197f1f37c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f37e8 sp=0x14197f1f37e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 34 gp=0x14197f604000 m=nil [GC worker (idle)]:
    runtime.gopark(0x125b862e8e02?, 0x3?, 0xe6?, 0x2d?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60a740 sp=0x14197f60a720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x14197f29d9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f60a7c8 sp=0x14197f60a740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f60a7e0 sp=0x14197f60a7c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60a7e8 sp=0x14197f60a7e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 24 gp=0x14197f5a32c0 m=nil [GC worker (idle)]:
    runtime.gopark(0x125b862118da?, 0x1?, 0x4d?, 0x9a?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f0f40 sp=0x14197f1f0f20 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x14197f29d9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f0fc8 sp=0x14197f1f0f40 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f0fe0 sp=0x14197f1f0fc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f0fe8 sp=0x14197f1f0fe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 4 gp=0x14197f133860 m=nil [GC worker (idle)]:
    runtime.gopark(0x125b7fe90140?, 0x3?, 0x20?, 0x3a?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f3f40 sp=0x14197f1f3f20 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x14197f29d9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f3fc8 sp=0x14197f1f3f40 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f3fe0 sp=0x14197f1f3fc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f3fe8 sp=0x14197f1f3fe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 35 gp=0x14197f6041e0 m=nil [GC worker (idle)]:
    runtime.gopark(0x125b862a34bd?, 0x3?, 0x4?, 0x15?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60af40 sp=0x14197f60af20 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x14197f29d9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f60afc8 sp=0x14197f60af40 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f60afe0 sp=0x14197f60afc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60afe8 sp=0x14197f60afe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 25 gp=0x14197f5a34a0 m=3 mp=0x14197f1a5808 [GC mark termination]:
    runtime.systemstack_switch()
    	/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x14197f1f16a0 sp=0x14197f1f1690 pc=0x1f3ebc8
    runtime.forEachP(...)
    	/usr/lib/golang/src/runtime/proc.go:2112
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x14197f1f1740 sp=0x14197f1f16a0 pc=0x1edbc34
    runtime.gcBgMarkWorker(0x14197f29d9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x14197f1f17c8 sp=0x14197f1f1740 pc=0x1edda88
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f17e0 sp=0x14197f1f17c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f17e8 sp=0x14197f1f17e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 5 gp=0x14197f133a40 m=nil [GC worker (idle)]:
    runtime.gopark(0x125b8624ea9b?, 0x1?, 0xfd?, 0x7a?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f4740 sp=0x14197f1f4720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x14197f29d9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f47c8 sp=0x14197f1f4740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f47e0 sp=0x14197f1f47c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f47e8 sp=0x14197f1f47e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 50 gp=0x14197f768f00 m=nil [select, locked to thread]:
    runtime.gopark(0x14197f60c7a8?, 0x2?, 0x1?, 0x8f?, 0x14197f60c794?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60c628 sp=0x14197f60c608 pc=0x1f38d2e
    runtime.selectgo(0x14197f60c7a8, 0x14197f60c790, 0x0?, 0x0, 0x0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f60c758 sp=0x14197f60c628 pc=0x1f152a5
    runtime.ensureSigM.func1()
    	/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x14197f60c7e0 sp=0x14197f60c758 pc=0x1f328e8
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60c7e8 sp=0x14197f60c7e0 pc=0x1f40801
    created by runtime.ensureSigM in goroutine 1
    	/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5
    
    goroutine 26 gp=0x14197f3b34a0 m=7 mp=0x14197f55b008 [syscall]:
    runtime.notetsleepg(0x82bdea0, 0xffffffffffffffff)
    	/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x14197f6077a0 sp=0x14197f607778 pc=0x1ece509
    os/signal.signal_recv()
    	/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x14197f6077c0 sp=0x14197f6077a0 pc=0x1f3b298
    os/signal.loop()
    	/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x14197f6077e0 sp=0x14197f6077c0 pc=0x2030553
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f6077e8 sp=0x14197f6077e0 pc=0x1f40801
    created by os/signal.Notify.func1.1 in goroutine 1
    	/usr/lib/golang/src/os/signal/signal.go:152 +0x1f
    
    goroutine 27 gp=0x14197f6894a0 m=nil [select]:
    runtime.gopark(0x14197f607f98?, 0x2?, 0x68?, 0x81?, 0x14197f607f8c?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f607e28 sp=0x14197f607e08 pc=0x1f38d2e
    runtime.selectgo(0x14197f607f98, 0x14197f607f88, 0x0?, 0x0, 0x0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f607f58 sp=0x14197f607e28 pc=0x1f152a5
    os/signal.NotifyContext.func1()
    	/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x14197f607fe0 sp=0x14197f607f58 pc=0x202ff9c
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f607fe8 sp=0x14197f607fe0 pc=0x1f40801
    created by os/signal.NotifyContext in goroutine 1
    	/usr/lib/golang/src/os/signal/signal.go:291 +0x158
    
    goroutine 224 gp=0x14197f689680 m=nil [semacquire]:
    runtime.gopark(0x7f80904adba8?, 0x1eed7e0?, 0xc0?, 0x68?, 0x4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419800b5b58 sp=0x1419800b5b38 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x1419800b5bc0 sp=0x1419800b5b58 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x1419800b5c60 sp=0x1419800b5bc0 pc=0x1edbb12
    runtime.gcAssistAlloc(0x14197f689680)
    	/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x1419800b5cf0 sp=0x1419800b5c60 pc=0x1ee06b6
    runtime.deductAssistCredit(0x7ffce51f9080?)
    	/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x1419800b5d10 sp=0x1419800b5cf0 pc=0x1ed10a6
    runtime.mallocgc(0x7fe00, 0x4af32a0, 0x1)
    	/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x1419800b5d40 sp=0x1419800b5d10 pc=0x1f35e07
    runtime.makeslice(0x14197fc9d0b0?, 0x141983a1fe00?, 0x20000?)
    	/usr/lib/golang/src/runtime/slice.go:117 +0x49 fp=0x1419800b5d68 sp=0x1419800b5d40 pc=0x1f3b489
    encoding/json.(*Decoder).refill(0x14197f658c80)
    	/usr/lib/golang/src/encoding/json/stream.go:161 +0xee fp=0x1419800b5db8 sp=0x1419800b5d68 pc=0x200d48e
    encoding/json.(*Decoder).readValue(0x14197f658c80)
    	/usr/lib/golang/src/encoding/json/stream.go:142 +0x85 fp=0x1419800b5e08 sp=0x1419800b5db8 pc=0x200d125
    encoding/json.(*Decoder).Decode(0x14197f658c80, {0x4cfb340, 0x14197ffa44c0})
    	/usr/lib/golang/src/encoding/json/stream.go:65 +0x75 fp=0x1419800b5e38 sp=0x1419800b5e08 pc=0x200cdf5
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readPrimaryTable(0x14197f8c54f0, 0x14197fb50000)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:264 +0x16e fp=0x1419800b5ec8 sp=0x1419800b5e38 pc=0x38d822e
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x14197f8c54f0)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:174 +0x189 fp=0x1419800b5f30 sp=0x1419800b5ec8 pc=0x38d7b89
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x14197f8c54f0, 0x14198007be90)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x1419800b5fc0 sp=0x1419800b5f30 pc=0x38d788c
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x1419800b5fe0 sp=0x1419800b5fc0 pc=0x38d751b
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1419800b5fe8 sp=0x1419800b5fe0 pc=0x1f40801
    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 183
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc
    
    goroutine 87 gp=0x14197fd3c000 m=nil [sync.WaitGroup.Wait]:
    runtime.gopark(0x14197fcf9e00?, 0x1ed0734?, 0x20?, 0x94?, 0x7f80917bdf06?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf9da0 sp=0x14197fcf9d80 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x14197f69cc50, 0x0, 0x1, 0x0, 0x19)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fcf9e08 sp=0x14197fcf9da0 pc=0x1f162b2
    sync.runtime_SemacquireWaitGroup(0x14197f86b2d8?, 0x0?)
    	/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x14197fcf9e40 sp=0x14197fcf9e08 pc=0x1f3a66e
    sync.(*WaitGroup).Wait(0x14197f69cc48)
    	/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x14197fcf9e68 sp=0x14197fcf9e40 pc=0x1f4f445
    golang.org/x/sync/errgroup.(*Group).Wait(0x14197f69cc40)
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x14197fcf9e88 sp=0x14197fcf9e68 pc=0x362507e
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x14197f521d80, {0x59f0b40, 0x14197f8c4b40}, {0x14197fcca000, 0x15, 0x6f63746669687372?}, 0x14197f1c02a0)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x14197fcf9f00 sp=0x14197fcf9e88 pc=0x392976a
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x14197fcf9f78 sp=0x14197fcf9f00 pc=0x392808e
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197fcf9fe0 sp=0x14197fcf9f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf9fe8 sp=0x14197fcf9fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 77 gp=0x14197fd3c1e0 m=nil [IO wait]:
    runtime.gopark(0x14197fb3e4a0?, 0x1edb176?, 0x80?, 0xed?, 0xb?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f202808 sp=0x14197f2027e8 pc=0x1f38d2e
    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)
    	/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x14197f202840 sp=0x14197f202808 pc=0x1ef9e57
    internal/poll.runtime_pollWait(0x7f8098109000, 0x72)
    	/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x14197f202860 sp=0x14197f202840 pc=0x1f375c5
    internal/poll.(*pollDesc).wait(0x14197f323a80?, 0x14197f34d300?, 0x0)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x14197f202888 sp=0x14197f202860 pc=0x1fbd167
    internal/poll.(*pollDesc).waitRead(...)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89
    internal/poll.(*FD).Read(0x14197f323a80, {0x14197f34d300, 0x1300, 0x1300})
    	/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x14197f202920 sp=0x14197f202888 pc=0x1fbe4ce
    net.(*netFD).Read(0x14197f323a80, {0x14197f34d300?, 0x14197f34d30a?, 0x5?})
    	/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x14197f202968 sp=0x14197f202920 pc=0x20798c5
    net.(*conn).Read(0x14197f1300d8, {0x14197f34d300?, 0x7f809057a868?, 0x7f80df53a6d8?})
    	/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x14197f2029b0 sp=0x14197f202968 pc=0x208b7a5
    crypto/tls.(*atLeastReader).Read(0x14197f821a10, {0x14197f34d300?, 0x14197f202a98?, 0x1efcc28?})
    	/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x14197f2029f8 sp=0x14197f2029b0 pc=0x2222ddb
    bytes.(*Buffer).ReadFrom(0x14197f8a57a8, {0x59a3680, 0x14197f821a10})
    	/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x14197f202a50 sp=0x14197f2029f8 pc=0x1fe9258
    crypto/tls.(*Conn).readFromUntil(0x14197f8a5508, {0x59a32c0, 0x14197f1300d8}, 0x222764c?)
    	/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x14197f202a88 sp=0x14197f202a50 pc=0x2222fbe
    crypto/tls.(*Conn).readRecordOrCCS(0x14197f8a5508, 0x0)
    	/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x14197f202c68 sp=0x14197f202a88 pc=0x222009b
    crypto/tls.(*Conn).readRecord(...)
    	/usr/lib/golang/src/crypto/tls/conn.go:588
    crypto/tls.(*Conn).Read(0x14197f8a5508, {0x14197fa26000, 0x1000, 0x4e55540?})
    	/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x14197f202cd0 sp=0x14197f202c68 pc=0x22267c5
    net/http.(*persistConn).Read(0x14197f8032c0, {0x14197fa26000?, 0x59a0ea0?, 0x81260e0?})
    	/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x14197f202d30 sp=0x14197f202cd0 pc=0x2319d07
    bufio.(*Reader).fill(0x14197f29b620)
    	/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f202d68 sp=0x14197f202d30 pc=0x20a6c63
    bufio.(*Reader).Peek(0x14197f29b620, 0x1)
    	/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x14197f202d80 sp=0x14197f202d68 pc=0x20a6df3
    net/http.(*persistConn).readLoop(0x14197f8032c0)
    	/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x14197f202fc8 sp=0x14197f202d80 pc=0x231a812
    net/http.(*Transport).dialConn.gowrap2()
    	/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x14197f202fe0 sp=0x14197f202fc8 pc=0x2319197
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f202fe8 sp=0x14197f202fe0 pc=0x1f40801
    created by net/http.(*Transport).dialConn in goroutine 9
    	/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac
    
    goroutine 7 gp=0x14197fe025a0 m=nil [sync.Mutex.Lock]:
    runtime.gopark(0x14197f9d3c68?, 0x14197f9d3c70?, 0x94?, 0x69?, 0x14197f9d3c78?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f9d3c00 sp=0x14197f9d3be0 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x14197fccaad4, 0x0, 0x3, 0x2, 0x16)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197f9d3c68 sp=0x14197f9d3c00 pc=0x1f162b2
    internal/sync.runtime_SemacquireMutex(0x1000?, 0xd0?, 0x1ed11a5?)
    	/usr/lib/golang/src/runtime/sema.go:95 +0x25 fp=0x14197f9d3ca0 sp=0x14197f9d3c68 pc=0x1f3a545
    internal/sync.(*Mutex).lockSlow(0x14197fccaad0)
    	/usr/lib/golang/src/internal/sync/mutex.go:149 +0x15d fp=0x14197f9d3cf0 sp=0x14197f9d3ca0 pc=0x1f4d31d
    internal/sync.(*Mutex).Lock(...)
    	/usr/lib/golang/src/internal/sync/mutex.go:70
    sync.(*Mutex).Lock(...)
    	/usr/lib/golang/src/sync/mutex.go:46
    net/http.(*http2clientConnReadLoop).streamByID(0x22babb5?, 0x1b, 0x1)
    	/usr/lib/golang/src/net/http/h2_bundle.go:10083 +0x5b fp=0x14197f9d3d48 sp=0x14197f9d3cf0 pc=0x22e10db
    net/http.(*http2clientConnReadLoop).processData(0x14197f9d3fa8, 0x14197fc9d200)
    	/usr/lib/golang/src/net/http/h2_bundle.go:9939 +0x45 fp=0x14197f9d3e18 sp=0x14197f9d3d48 pc=0x22e05e5
    net/http.(*http2clientConnReadLoop).run(0x14197f9d3fa8)
    	/usr/lib/golang/src/net/http/h2_bundle.go:9584 +0x4b6 fp=0x14197f9d3f60 sp=0x14197f9d3e18 pc=0x22de536
    net/http.(*http2ClientConn).readLoop(0x14197fccaa80)
    	/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x14197f9d3fc8 sp=0x14197f9d3f60 pc=0x22dd532
    net/http.(*http2Transport).newClientConn.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x14197f9d3fe0 sp=0x14197f9d3fc8 pc=0x22d7057
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f9d3fe8 sp=0x14197f9d3fe0 pc=0x1f40801
    created by net/http.(*http2Transport).newClientConn in goroutine 6
    	/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5
    
    goroutine 78 gp=0x14197fe02960 m=nil [select]:
    runtime.gopark(0x14197f9f7f48?, 0x2?, 0x0?, 0xb6?, 0x14197f9f7ef4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf2d88 sp=0x14197fcf2d68 pc=0x1f38d2e
    runtime.selectgo(0x14197fcf2f48, 0x14197f9f7ef0, 0x14197f1cd0c0?, 0x0, 0x14197f89c2a0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197fcf2eb8 sp=0x14197fcf2d88 pc=0x1f152a5
    net/http.(*persistConn).writeLoop(0x14197f8032c0)
    	/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x14197fcf2fc8 sp=0x14197fcf2eb8 pc=0x231c426
    net/http.(*Transport).dialConn.gowrap3()
    	/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x14197fcf2fe0 sp=0x14197fcf2fc8 pc=0x2319157
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf2fe8 sp=0x14197fcf2fe0 pc=0x1f40801
    created by net/http.(*Transport).dialConn in goroutine 9
    	/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe
    
    goroutine 163 gp=0x14197f7694a0 m=nil [select]:
    runtime.gopark(0x14197fdd3f08?, 0x6?, 0xa0?, 0x94?, 0x14197fdd3da4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f205c08 sp=0x14197f205be8 pc=0x1f38d2e
    runtime.selectgo(0x14197f205f08, 0x14197fdd3d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f205d38 sp=0x14197f205c08 pc=0x1f152a5
    net/http.(*http2clientStream).writeRequest(0x14197f37f380, 0x14197f49e140, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x14197f205f90 sp=0x14197f205d38 pc=0x22da6a8
    net/http.(*http2clientStream).doRequest(0x14197f37f380, 0x14197f546100?, 0x14197f609798?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x14197f205fb8 sp=0x14197f205f90 pc=0x22d99d8
    net/http.(*http2ClientConn).roundTrip.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197f205fe0 sp=0x14197f205fb8 pc=0x22d999f
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f205fe8 sp=0x14197f205fe0 pc=0x1f40801
    created by net/http.(*http2ClientConn).roundTrip in goroutine 184
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470
    
    goroutine 88 gp=0x14197f769a40 m=nil [semacquire]:
    runtime.gopark(0x141980119958?, 0x1ed3e05?, 0xe8?, 0xd2?, 0x7f80904ad2e8?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x141980119908 sp=0x1419801198e8 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x141980119970 sp=0x141980119908 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x141980119a10 sp=0x141980119970 pc=0x1edbb12
    runtime.gcAssistAlloc(0x14197f769a40)
    	/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x141980119aa0 sp=0x141980119a10 pc=0x1ee06b6
    runtime.deductAssistCredit(0x141983b00000?)
    	/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x141980119ac0 sp=0x141980119aa0 pc=0x1ed10a6
    runtime.mallocgc(0x7edd, 0x0, 0x0)
    	/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x141980119af0 sp=0x141980119ac0 pc=0x1f35e07
    runtime.slicebytetostring(0x4e537a0?, 0x141983b00000, 0x7edd)
    	/usr/lib/golang/src/runtime/string.go:170 +0x77 fp=0x141980119b20 sp=0x141980119af0 pc=0x1f1e397
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x14197f8c4b40}, 0x14197f1c03f0, 0x14197f531740)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:322 +0x91b fp=0x141980119f48 sp=0x141980119b20 pc=0x3920bfb
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x141980119f78 sp=0x141980119f48 pc=0x392800c
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x141980119fe0 sp=0x141980119f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x141980119fe8 sp=0x141980119fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 110 gp=0x14197fd3cb40 m=nil [sync.Mutex.Lock]:
    runtime.gopark(0x22b8a73?, 0x14197f7e1000?, 0xb0?, 0x4c?, 0x8160a40?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1c7510 sp=0x14197f1c74f0 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x14197fccaad4, 0x0, 0x3, 0x2, 0x16)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197f1c7578 sp=0x14197f1c7510 pc=0x1f162b2
    internal/sync.runtime_SemacquireMutex(0x14197f1c75d0?, 0x66?, 0x61f?)
    	/usr/lib/golang/src/runtime/sema.go:95 +0x25 fp=0x14197f1c75b0 sp=0x14197f1c7578 pc=0x1f3a545
    internal/sync.(*Mutex).lockSlow(0x14197fccaad0)
    	/usr/lib/golang/src/internal/sync/mutex.go:149 +0x15d fp=0x14197f1c7600 sp=0x14197f1c75b0 pc=0x1f4d31d
    internal/sync.(*Mutex).Lock(...)
    	/usr/lib/golang/src/internal/sync/mutex.go:70
    sync.(*Mutex).Lock(...)
    	/usr/lib/golang/src/sync/mutex.go:46
    net/http.http2transportResponseBody.Read({0x1f3ec12?}, {0x14198313d000?, 0x1eed797?, 0x14197f1c76b0?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:9875 +0x209 fp=0x14197f1c7688 sp=0x14197f1c7600 pc=0x22dff89
    bufio.(*Reader).fill(0x14197f3a00c0)
    	/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f1c76c0 sp=0x14197f1c7688 pc=0x20a6c63
    bufio.(*Reader).ReadByte(0x14197f3a00c0)
    	/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x14197f1c76d8 sp=0x14197f1c76c0 pc=0x20a73c7
    compress/flate.(*decompressor).moreBits(0x141980059908)
    	/usr/lib/golang/src/compress/flate/inflate.go:697 +0x24 fp=0x14197f1c7710 sp=0x14197f1c76d8 pc=0x2288824
    compress/flate.(*decompressor).nextBlock(0x141980059908)
    	/usr/lib/golang/src/compress/flate/inflate.go:304 +0x28 fp=0x14197f1c7728 sp=0x14197f1c7710 pc=0x2286ba8
    compress/flate.(*decompressor).Read(0x141980059908, {0x14198314e000, 0x1000, 0x7f8090ef0f00?})
    	/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x14197f1c7758 sp=0x14197f1c7728 pc=0x2286dbb
    compress/gzip.(*Reader).Read(0x14197f672dc8, {0x14198314e000, 0x1000, 0x1000})
    	/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x14197f1c77e8 sp=0x14197f1c7758 pc=0x228c838
    github.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x410d0fa76c5d7d40?, {0x14198314e000?, 0x7f8090ef0d20?, 0x52ffb40?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x14197f1c7818 sp=0x14197f1c77e8 pc=0x38b33e2
    bufio.(*Reader).fill(0x14197f3a0120)
    	/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f1c7850 sp=0x14197f1c7818 pc=0x20a6c63
    bufio.(*Reader).Peek(0x14197f3a0120, 0x1)
    	/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x14197f1c7868 sp=0x14197f1c7850 pc=0x20a6df3
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.validateJsonResponse(0x14197f3a0120)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:38 +0x25 fp=0x14197f1c78b8 sp=0x14197f1c7868 pc=0x38d6be5
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.newFrameReader({0x59db110, 0x14197fe1e040}, {0x59f0b40, 0x14197f748140})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:26 +0x188 fp=0x14197f1c7960 sp=0x14197f1c78b8 pc=0x38d6a08
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset({0x59f0bb0?, 0x14197f4797a0?}, {0x59db110, 0x14197fe1e040}, 0x1, 0x3e8, 0x1)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:66 +0x19e fp=0x14197f1c79e0 sp=0x14197f1c7960 pc=0x38d737e
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x14197f521d40, {0x59f0bb0, 0x14197f4797a0}, {0x14197f49a330, 0xb}, 0x14197fe1e700, {0x0?, 0x2?, 0x0?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:167 +0x2b7 fp=0x14197f1c7a78 sp=0x14197f1c79e0 pc=0x38e4f17
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197f7a2600}, 0x14197f1c02a0)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x14197f1c7d60 sp=0x14197f1c7a78 pc=0x391b5c5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x14197f521d80, {0x59f0b40, 0x14197fdd8140}, {0x5a00340, 0x14197fc1e5d0}, 0x14197f1c02a0)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:98 +0x366 fp=0x14197f1c7e88 sp=0x14197f1c7d60 pc=0x3929ec6
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14197f1c7f78 sp=0x14197f1c7e88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197f1c7fe0 sp=0x14197f1c7f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1c7fe8 sp=0x14197f1c7fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 89 gp=0x14197fd82960 m=nil [select]:
    runtime.gopark(0x1419800b7d00?, 0x2?, 0xd7?, 0xfd?, 0x1419800b7c74?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419800b7ac8 sp=0x1419800b7aa8 pc=0x1f38d2e
    runtime.selectgo(0x1419800b7d00, 0x1419800b7c70, 0x1419801521c0?, 0x0, 0x14197f5daba0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x1419800b7bf8 sp=0x1419800b7ac8 pc=0x1f152a5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x14197f521d80?, {0x59f0b40, 0x14197f8c4b40}, 0x14197f1c02a0, 0x14197f1c03f0)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x1419800b7f10 sp=0x1419800b7bf8 pc=0x392836e
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x1419800b7f78 sp=0x1419800b7f10 pc=0x3927f3d
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x1419800b7fe0 sp=0x1419800b7f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1419800b7fe8 sp=0x1419800b7fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 175 gp=0x14197fd3d2c0 m=nil [sync.Cond.Wait]:
    runtime.gopark(0x22c2d5b?, 0x14197fe1c080?, 0x0?, 0xe0?, 0x14197f777000?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14198012b978 sp=0x14198012b958 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    sync.runtime_notifyListWait(0x14197f37f3c8, 0x3c)
    	/usr/lib/golang/src/runtime/sema.go:617 +0x1b3 fp=0x14198012b9c8 sp=0x14198012b978 pc=0x1f3a8f3
    sync.(*Cond).Wait(0x1000?)
    	/usr/lib/golang/src/sync/cond.go:71 +0x73 fp=0x14198012ba00 sp=0x14198012b9c8 pc=0x1f4d673
    net/http.(*http2pipe).Read(0x14197f37f3b0, {0x14197ff7e000, 0x1000, 0x1000})
    	/usr/lib/golang/src/net/http/h2_bundle.go:3928 +0xdb fp=0x14198012ba68 sp=0x14198012ba00 pc=0x22c2c1b
    net/http.http2transportResponseBody.Read({0x4a54?}, {0x14197ff7e000?, 0x14198012bb80?, 0x2287949?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:9852 +0x59 fp=0x14198012baf0 sp=0x14198012ba68 pc=0x22dfdd9
    bufio.(*Reader).fill(0x14197fcaa000)
    	/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14198012bb28 sp=0x14198012baf0 pc=0x20a6c63
    bufio.(*Reader).ReadByte(0x14197fcaa000)
    	/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x14198012bb40 sp=0x14198012bb28 pc=0x20a73c7
    compress/flate.(*decompressor).moreBits(0x141980056008)
    	/usr/lib/golang/src/compress/flate/inflate.go:697 +0x24 fp=0x14198012bb78 sp=0x14198012bb40 pc=0x2288824
    compress/flate.(*decompressor).nextBlock(0x141980056008)
    	/usr/lib/golang/src/compress/flate/inflate.go:304 +0x28 fp=0x14198012bb90 sp=0x14198012bb78 pc=0x2286ba8
    compress/flate.(*decompressor).Read(0x141980056008, {0x141980034b13, 0x4ed, 0xb13?})
    	/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x14198012bbc0 sp=0x14198012bb90 pc=0x2286dbb
    compress/gzip.(*Reader).Read(0x141980000b08, {0x141980034b13, 0x4ed, 0x4ed})
    	/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x14198012bc50 sp=0x14198012bbc0 pc=0x228c838
    github.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x14198012bcf0?, {0x141980034b13?, 0x1000?, 0x0?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x14198012bc80 sp=0x14198012bc50 pc=0x38b33e2
    bufio.(*Reader).fill(0x141980807010)
    	/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14198012bcb8 sp=0x14198012bc80 pc=0x20a6c63
    bufio.(*Reader).ReadSlice(0x141980807010, 0xa)
    	/usr/lib/golang/src/bufio/bufio.go:377 +0x29 fp=0x14198012bd00 sp=0x14198012bcb8 pc=0x20a7769
    bufio.(*Reader).collectFragments(0x141980807010, 0xa)
    	/usr/lib/golang/src/bufio/bufio.go:452 +0x85 fp=0x14198012bde0 sp=0x14198012bd00 pc=0x20a7b85
    bufio.(*Reader).ReadBytes(0x4bbc5a0?, 0xa0?)
    	/usr/lib/golang/src/bufio/bufio.go:479 +0x17 fp=0x14198012be60 sp=0x14198012bde0 pc=0x20a7e77
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.(*frameReader).advance(0x141980807000)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:64 +0x45 fp=0x14198012beb0 sp=0x14198012be60 pc=0x38d6d45
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readRoutine(0x141980807000, 0x14197f8c8370)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:85 +0x51 fp=0x14198012bfc0 sp=0x14198012beb0 pc=0x38d7591
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap2()
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x1b fp=0x14198012bfe0 sp=0x14198012bfc0 pc=0x38d74db
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14198012bfe8 sp=0x14198012bfe0 pc=0x1f40801
    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 184
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x24a
    
    goroutine 183 gp=0x14197fd3d860 m=nil [semacquire]:
    runtime.gopark(0x7f8090f36c08?, 0x1eed7e0?, 0x80?, 0x43?, 0x2?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fac5208 sp=0x14197fac51e8 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fac5270 sp=0x14197fac5208 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x14197fac5310 sp=0x14197fac5270 pc=0x1edbb12
    runtime.gcAssistAlloc(0x14197fd3d860)
    	/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x14197fac53a0 sp=0x14197fac5310 pc=0x1ee06b6
    runtime.deductAssistCredit(0x14197fac53f0?)
    	/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x14197fac53c0 sp=0x14197fac53a0 pc=0x1ed10a6
    runtime.mallocgc(0x5500, 0x0, 0x0)
    	/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x14197fac53f0 sp=0x14197fac53c0 pc=0x1f35e07
    runtime.growslice(0x141983adc000, 0x14197fac54a8?, 0x1ed3be5?, 0x82bed80?, 0x1ed0417?)
    	/usr/lib/golang/src/runtime/slice.go:265 +0x573 fp=0x14197fac5460 sp=0x14197fac53f0 pc=0x1f3ba93
    strings.(*Builder).Write(0x14197f569ea0, {0x1419839fd000, 0x1000, 0x1000000010765?})
    	/usr/lib/golang/src/strings/builder.go:89 +0x90 fp=0x14197fac54b0 sp=0x14197fac5460 pc=0x1ff0cb0
    bufio.(*Writer).Flush(0x14197fac5698)
    	/usr/lib/golang/src/bufio/bufio.go:642 +0x52 fp=0x14197fac54f8 sp=0x14197fac54b0 pc=0x20a8832
    bufio.(*Writer).WriteString(0x14197fac5698, {0x553a4f6?, 0x553a9e6?})
    	/usr/lib/golang/src/bufio/bufio.go:766 +0x165 fp=0x14197fac5568 sp=0x14197fac54f8 pc=0x20a8f65
    encoding/csv.(*Writer).Write(0x14197fac5688, {0x14198384d480?, 0x4, 0x141983909500?})
    	/usr/lib/golang/src/encoding/csv/writer.go:92 +0x2b4 fp=0x14197fac55d0 sp=0x14197fac5568 pc=0x20ad3d4
    github.com/Azure/azure-kusto-go/azkustodata/query.(*row).String(0x5e?)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/row_impl.go:101 +0x21f fp=0x14197fac56e8 sp=0x14197fac55d0 pc=0x38cf67f
    fmt.(*pp).handleMethods(0x14197fc20000, 0x7fac58f0?)
    	/usr/lib/golang/src/fmt/print.go:674 +0x2f4 fp=0x14197fac58a8 sp=0x14197fac56e8 pc=0x1fdd1d4
    fmt.(*pp).printArg(0x14197fc20000, {0x549a3a0, 0x14197fd92e80}, 0x76)
    	/usr/lib/golang/src/fmt/print.go:757 +0x42e fp=0x14197fac5938 sp=0x14197fac58a8 pc=0x1fdd9ee
    fmt.(*pp).doPrintf(0x14197fc20000, {0x553a400, 0x2}, {0x14197fac5d08, 0x1, 0x1})
    	/usr/lib/golang/src/fmt/print.go:1075 +0x3ec fp=0x14197fac5a20 sp=0x14197fac5938 pc=0x1fe04ec
    fmt.Sprintf({0x553a400, 0x2}, {0x14197fac5d08, 0x1, 0x1})
    	/usr/lib/golang/src/fmt/print.go:239 +0x53 fp=0x14197fac5a78 sp=0x14197fac5a20 pc=0x1fda6b3
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197fc4b200}, 0x14197f1c02a0)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:160 +0x8d8 fp=0x14197fac5d60 sp=0x14197fac5a78 pc=0x391bad8
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x14197fdd8140?}, {0x5a00300?, 0x14197fc4b200?}, 0x0?)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x14197fac5e88 sp=0x14197fac5d60 pc=0x3929c16
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14197fac5f78 sp=0x14197fac5e88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197fac5fe0 sp=0x14197fac5f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fac5fe8 sp=0x14197fac5fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 184 gp=0x14197fd3da40 m=nil [chan receive]:
    runtime.gopark(0x14197f3c29a0?, 0x2034990?, 0x10?, 0xea?, 0x59f0ad0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14198011f9d8 sp=0x14198011f9b8 pc=0x1f38d2e
    runtime.chanrecv(0x14197f3fa700, 0x14198011fcb8, 0x1)
    	/usr/lib/golang/src/runtime/chan.go:667 +0x4ae fp=0x14198011fa50 sp=0x14198011f9d8 pc=0x1ec7d2e
    runtime.chanrecv2(0x5a09ed0?, 0x14197f530f90?)
    	/usr/lib/golang/src/runtime/chan.go:514 +0x12 fp=0x14198011fa78 sp=0x14198011fa50 pc=0x1ec7872
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197fc4b340}, 0x14197f1c02a0)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:141 +0x94d fp=0x14198011fd60 sp=0x14198011fa78 pc=0x391bb4d
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x74666968736e6570?, {0x59f0b40?, 0x14197fdd8140?}, {0x5a00300?, 0x14197fc4b340?}, 0x6d736f63222c2273?)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x14198011fe88 sp=0x14198011fd60 pc=0x3929c16
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14198011ff78 sp=0x14198011fe88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14198011ffe0 sp=0x14198011ff78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14198011ffe8 sp=0x14198011ffe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 135 gp=0x14197fb465a0 m=nil [semacquire]:
    runtime.gopark(0x14197f454000?, 0x22dc6c0?, 0xa0?, 0x82?, 0x1419808322a0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fe99ce8 sp=0x14197fe99cc8 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fe99d50 sp=0x14197fe99ce8 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x14197fe99df0 sp=0x14197fe99d50 pc=0x1edbb12
    runtime.gcAssistAlloc(0x14197fb465a0)
    	/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x14197fe99e80 sp=0x14197fe99df0 pc=0x1ee06b6
    runtime.deductAssistCredit(0x0?)
    	/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x14197fe99ea0 sp=0x14197fe99e80 pc=0x1ed10a6
    runtime.mallocgc(0x70, 0x0, 0x1)
    	/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x14197fe99ed0 sp=0x14197fe99ea0 pc=0x1f35e07
    runtime.makechan(0x0?, 0x0)
    	/usr/lib/golang/src/runtime/chan.go:99 +0x126 fp=0x14197fe99f10 sp=0x14197fe99ed0 pc=0x1ec6826
    net/http.(*http2clientStream).cleanupWriteRequest(0x14197f2f1200, {0x0, 0x0})
    	/usr/lib/golang/src/net/http/h2_bundle.go:8951 +0xd6 fp=0x14197fe99f90 sp=0x14197fe99f10 pc=0x22db1b6
    net/http.(*http2clientStream).doRequest(0x14197f2f1200, 0x14197f7480a0?, 0x14197f4d0020?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8719 +0x28 fp=0x14197fe99fb8 sp=0x14197fe99f90 pc=0x22d99e8
    net/http.(*http2ClientConn).roundTrip.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197fe99fe0 sp=0x14197fe99fb8 pc=0x22d999f
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fe99fe8 sp=0x14197fe99fe0 pc=0x1f40801
    created by net/http.(*http2ClientConn).roundTrip in goroutine 110
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470
    
    goroutine 90 gp=0x14197fd82b40 m=nil [select]:
    runtime.gopark(0x14197ff1df08?, 0x6?, 0x40?, 0x2b?, 0x14197ff1dda4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f203c08 sp=0x14197f203be8 pc=0x1f38d2e
    runtime.selectgo(0x14197f203f08, 0x14197ff1dd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f203d38 sp=0x14197f203c08 pc=0x1f152a5
    net/http.(*http2clientStream).writeRequest(0x14197f2f0f00, 0x14197fb50140, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x14197f203f90 sp=0x14197f203d38 pc=0x22da6a8
    net/http.(*http2clientStream).doRequest(0x14197f2f0f00, 0x0?, 0x14197f1f1f98?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x14197f203fb8 sp=0x14197f203f90 pc=0x22d99d8
    net/http.(*http2ClientConn).roundTrip.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197f203fe0 sp=0x14197f203fb8 pc=0x22d999f
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f203fe8 sp=0x14197f203fe0 pc=0x1f40801
    created by net/http.(*http2ClientConn).roundTrip in goroutine 183
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470
    
    goroutine 225 gp=0x14197fd834a0 m=nil [sync.Cond.Wait]:
    runtime.gopark(0x22b8a73?, 0x141980390000?, 0x0?, 0x60?, 0xe?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419801298f8 sp=0x1419801298d8 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    sync.runtime_notifyListWait(0x14197f2f0f48, 0x3c)
    	/usr/lib/golang/src/runtime/sema.go:617 +0x1b3 fp=0x141980129948 sp=0x1419801298f8 pc=0x1f3a8f3
    sync.(*Cond).Wait(0x14197fbb1640?)
    	/usr/lib/golang/src/sync/cond.go:71 +0x73 fp=0x141980129980 sp=0x141980129948 pc=0x1f4d673
    net/http.(*http2pipe).Read(0x14197f2f0f30, {0x14197fd98000, 0x1000, 0x1000})
    	/usr/lib/golang/src/net/http/h2_bundle.go:3928 +0xdb fp=0x1419801299e8 sp=0x141980129980 pc=0x22c2c1b
    net/http.http2transportResponseBody.Read({0x141980129a60?}, {0x14197fd98000?, 0x20a6c63?, 0x1f3ec12?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:9852 +0x59 fp=0x141980129a70 sp=0x1419801299e8 pc=0x22dfdd9
    bufio.(*Reader).fill(0x14197fc90180)
    	/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x141980129aa8 sp=0x141980129a70 pc=0x20a6c63
    bufio.(*Reader).ReadByte(0x14197fc90180)
    	/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x141980129ac0 sp=0x141980129aa8 pc=0x20a73c7
    compress/flate.(*decompressor).huffSym(0x141980058608, 0x141980058638)
    	/usr/lib/golang/src/compress/flate/inflate.go:720 +0x7e fp=0x141980129b10 sp=0x141980129ac0 pc=0x228893e
    compress/flate.(*decompressor).huffmanBlock(0x82bed80?)
    	/usr/lib/golang/src/compress/flate/inflate.go:495 +0x45 fp=0x141980129b90 sp=0x141980129b10 pc=0x2287605
    compress/flate.(*decompressor).Read(0x141980058608, {0x14197fd99cea, 0x316, 0xcea?})
    	/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x141980129bc0 sp=0x141980129b90 pc=0x2286dbb
    compress/gzip.(*Reader).Read(0x14197f3d8848, {0x14197fd99cea, 0x316, 0x316})
    	/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x141980129c50 sp=0x141980129bc0 pc=0x228c838
    github.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x141980129cf0?, {0x14197fd99cea?, 0x1000?, 0x0?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x141980129c80 sp=0x141980129c50 pc=0x38b33e2
    bufio.(*Reader).fill(0x14197f546110)
    	/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x141980129cb8 sp=0x141980129c80 pc=0x20a6c63
    bufio.(*Reader).ReadSlice(0x14197f546110, 0xa)
    	/usr/lib/golang/src/bufio/bufio.go:377 +0x29 fp=0x141980129d00 sp=0x141980129cb8 pc=0x20a7769
    bufio.(*Reader).collectFragments(0x14197f546110, 0xa)
    	/usr/lib/golang/src/bufio/bufio.go:452 +0x85 fp=0x141980129de0 sp=0x141980129d00 pc=0x20a7b85
    bufio.(*Reader).ReadBytes(0x4bbc5a0?, 0xa0?)
    	/usr/lib/golang/src/bufio/bufio.go:479 +0x17 fp=0x141980129e60 sp=0x141980129de0 pc=0x20a7e77
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.(*frameReader).advance(0x14197f546100)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:64 +0x45 fp=0x141980129eb0 sp=0x141980129e60 pc=0x38d6d45
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readRoutine(0x14197f546100, 0x14197f8c54f0)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:85 +0x51 fp=0x141980129fc0 sp=0x141980129eb0 pc=0x38d7591
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap2()
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x1b fp=0x141980129fe0 sp=0x141980129fc0 pc=0x38d74db
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x141980129fe8 sp=0x141980129fe0 pc=0x1f40801
    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 183
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x24a
    
    goroutine 174 gp=0x14197ff64000 m=nil [select]:
    runtime.gopark(0x1419800b9df8?, 0x2?, 0x0?, 0x0?, 0x1419800b9db4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf3c50 sp=0x14197fcf3c30 pc=0x1f38d2e
    runtime.selectgo(0x14197fcf3df8, 0x1419800b9db0, 0x59d87c8?, 0x0, 0x14197f2980e0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197fcf3d80 sp=0x14197fcf3c50 pc=0x1f152a5
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.nextFrame(0x14197f8c8370)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:198 +0x96 fp=0x14197fcf3e38 sp=0x14197fcf3d80 pc=0x38d7d16
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readPrimaryTable(0x14197f8c8370, 0x14197fb50000)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:258 +0xb1 fp=0x14197fcf3ec8 sp=0x14197fcf3e38 pc=0x38d8171
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x14197f8c8370)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:174 +0x189 fp=0x14197fcf3f30 sp=0x14197fcf3ec8 pc=0x38d7b89
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x14197f8c8370, 0x14197f59aee0)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x14197fcf3fc0 sp=0x14197fcf3f30 pc=0x38d788c
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x14197fcf3fe0 sp=0x14197fcf3fc0 pc=0x38d751b
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf3fe8 sp=0x14197fcf3fe0 pc=0x1f40801
    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 184
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc
    
    rax    0x0
    rbx    0x7f808bfff640
    rcx    0x7f80df60594c
    rdx    0x6
    rdi    0x109fd
    rsi    0x10a00
    rbp    0x10a00
    rsp    0x7f808bffe790
    r8     0x7f808bffe860
    r9     0x7f80df72b4e0
    r10    0x8
    r11    0x246
    r12    0x6
    r13    0x7f808bffeb60
    r14    0x14197f1334a0
    r15    0xffffffffffffffff
    rip    0x7f80df60594c
    rflags 0x246
    cs     0x33
    fs     0x0
    gs     0x0
    
      query-infra-svc: command failed: exit status 2
    output: runtime/cgo: pthread_create failed: Resource temporarily unavailable
    SIGABRT: abort
    PC=0x7f7ef9baa94c m=3 sigcode=18446744073709551610
    
    goroutine 0 gp=0x170ac5c1f0e0 m=3 mp=0x170ac5c91808 [idle]:
    runtime: g 0 gp=0x170ac5c1f0e0: unknown pc 0x7f7ef9baa94c
    stack: frame={sp:0x7f7eabffe790, fp:0x0} stack=[0x7f7eab7ff1e0,0x7f7eabffede0)
                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef
    00007f7eabffe690: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6a0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6b0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6c0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6d0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6e0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6f0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe700: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe710: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe720: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe730: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe740: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe750: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe760: 00000000 00000000  00007f7e f9bb7fa9  ............~...
    00007f7eabffe770: 00000000 00000000  00000000 00000011  ................
    00007f7eabffe780: 00000000 00000130  00007f7e f9baa93e  0.......>...~...
    00007f7eabffe790: 00000000 00000120  00000000 00000000   ...............
                      ^ SP
    00007f7eabffe7a0: 00000000 00000013  00000000 00000000  ................
    00007f7eabffe7b0: 00000000 00000000  00007f7e f9c2dcff  ............~...
    00007f7eabffe7c0: 00007f7e 7bfff910  00007f7e f9c2dcb1  ...{~.......~...
    00007f7eabffe7d0: 00007f7e 7bfff640  00007f7e abffeaa0  @..{~.......~...
    00007f7eabffe7e0: 00007f7e abffe90e  00007f7e abffe90f  ....~.......~...
    00007f7eabffe7f0: 00007f7e 7bfff640  00007f7e f9ba87f5  @..{~.......~...
    00007f7eabffe800: 00007f7e a4000020  00000000 00000120   ...~... .......
    00007f7eabffe810: 00000000 003d0f00  176a34c4 bac87500  ..=......u...4j.
    00007f7eabffe820: 00007f7e abfff640  00000000 00000006  @...~...........
    00007f7eabffe830: 00007f7e a4000f30  00007f7e abffeb60  0...~...`...~...
    00007f7eabffe840: 0000170a c5c1f0e0  00007f7e f9b5d646  ........F...~...
    00007f7eabffe850: 00007f7e f9d1ae90  00007f7e f9b477f3  ....~....w..~...
    00007f7eabffe860: 00000000 00000020  176a34c4 bac87500   ........u...4j.
    00007f7eabffe870: 00007f7e 7b7ff000  00007f7e f9c1cac4  ...{~.......~...
    00007f7eabffe880: 00000000 00000000  00000000 0000000d  ................
    runtime: g 0 gp=0x170ac5c1f0e0: unknown pc 0x7f7ef9baa94c
    stack: frame={sp:0x7f7eabffe790, fp:0x0} stack=[0x7f7eab7ff1e0,0x7f7eabffede0)
                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef
    00007f7eabffe690: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6a0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6b0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6c0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6d0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6e0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe6f0: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe700: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe710: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe720: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe730: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe740: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe750: 00000000 00000000  00000000 00000000  ................
    00007f7eabffe760: 00000000 00000000  00007f7e f9bb7fa9  ............~...
    00007f7eabffe770: 00000000 00000000  00000000 00000011  ................
    00007f7eabffe780: 00000000 00000130  00007f7e f9baa93e  0.......>...~...
    00007f7eabffe790: 00000000 00000120  00000000 00000000   ...............
                      ^ SP
    00007f7eabffe7a0: 00000000 00000013  00000000 00000000  ................
    00007f7eabffe7b0: 00000000 00000000  00007f7e f9c2dcff  ............~...
    00007f7eabffe7c0: 00007f7e 7bfff910  00007f7e f9c2dcb1  ...{~.......~...
    00007f7eabffe7d0: 00007f7e 7bfff640  00007f7e abffeaa0  @..{~.......~...
    00007f7eabffe7e0: 00007f7e abffe90e  00007f7e abffe90f  ....~.......~...
    00007f7eabffe7f0: 00007f7e 7bfff640  00007f7e f9ba87f5  @..{~.......~...
    00007f7eabffe800: 00007f7e a4000020  00000000 00000120   ...~... .......
    00007f7eabffe810: 00000000 003d0f00  176a34c4 bac87500  ..=......u...4j.
    00007f7eabffe820: 00007f7e abfff640  00000000 00000006  @...~...........
    00007f7eabffe830: 00007f7e a4000f30  00007f7e abffeb60  0...~...`...~...
    00007f7eabffe840: 0000170a c5c1f0e0  00007f7e f9b5d646  ........F...~...
    00007f7eabffe850: 00007f7e f9d1ae90  00007f7e f9b477f3  ....~....w..~...
    00007f7eabffe860: 00000000 00000020  176a34c4 bac87500   ........u...4j.
    00007f7eabffe870: 00007f7e 7b7ff000  00007f7e f9c1cac4  ...{~.......~...
    00007f7eabffe880: 00000000 00000000  00000000 0000000d  ................
    
    goroutine 1 gp=0x170ac5c1e1e0 m=nil [sync.WaitGroup.Wait]:
    runtime.gopark(0x170ac5cb27b8?, 0x2022a4d?, 0x70?, 0x8e?, 0x170ac5cb27e8?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cb27a0 sp=0x170ac5cb2780 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x170ac65f8210, 0x0, 0x1, 0x0, 0x19)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x170ac5cb2808 sp=0x170ac5cb27a0 pc=0x1f162b2
    sync.runtime_SemacquireWaitGroup(0x170ac6709950?, 0xe0?)
    	/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x170ac5cb2840 sp=0x170ac5cb2808 pc=0x1f3a66e
    sync.(*WaitGroup).Wait(0x170ac65f8208)
    	/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x170ac5cb2868 sp=0x170ac5cb2840 pc=0x1f4f445
    golang.org/x/sync/errgroup.(*Group).Wait(0x170ac65f8200)
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x170ac5cb2888 sp=0x170ac5cb2868 pc=0x362507e
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0}, {0x170ac626a1e0, 0x6, 0x6})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x170ac5cb2948 sp=0x170ac5cb2888 pc=0x3927e05
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).gatherInfraLogs(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:595 +0xadd fp=0x170ac5cb2d30 sp=0x170ac5cb2948 pc=0x392799d
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:333 +0x533 fp=0x170ac5cb3908 sp=0x170ac5cb2d30 pc=0x3921bf3
    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedInfraQueryOptions).RunInfra(0x170ac5c2c108, {0x59f0b08, 0x170ac64108d0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:83 +0x71f fp=0x170ac5cb3b30 sp=0x170ac5cb3908 pc=0x393887f
    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawInfraQueryOptions).Run(0x0?, {0x59f0b08, 0x170ac64108d0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_options.go:109 +0x54 fp=0x170ac5cb3b58 sp=0x170ac5cb3b30 pc=0x3939134
    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryInfraCommand.func1(0x170ac66f5508?, {0x170ac5d40960?, 0x4?, 0x553b0e2?})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:45 +0x28 fp=0x170ac5cb3b80 sp=0x170ac5cb3b58 pc=0x3938128
    github.com/spf13/cobra.(*Command).execute(0x170ac66f5508, {0x170ac5e36100, 0xa, 0xa})
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x170ac5cb3d28 sp=0x170ac5cb3b80 pc=0x212e614
    github.com/spf13/cobra.(*Command).ExecuteC(0x170ac6228608)
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x170ac5cb3e18 sp=0x170ac5cb3d28 pc=0x212efa5
    github.com/spf13/cobra.(*Command).Execute(...)
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071
    github.com/spf13/cobra.(*Command).ExecuteContext(...)
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064
    main.main()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x170ac5cb3f48 sp=0x170ac5cb3e18 pc=0x47ddd6a
    runtime.main()
    	/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x170ac5cb3fe0 sp=0x170ac5cb3f48 pc=0x1f01975
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cb3fe8 sp=0x170ac5cb3fe0 pc=0x1f40801
    
    goroutine 2 gp=0x170ac5c1ed20 m=nil [force gc (idle)]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdefa8 sp=0x170ac5cdef88 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.forcegchelper()
    	/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x170ac5cdefe0 sp=0x170ac5cdefa8 pc=0x1f01c93
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdefe8 sp=0x170ac5cdefe0 pc=0x1f40801
    created by runtime.init.7 in goroutine 1
    	/usr/lib/golang/src/runtime/proc.go:363 +0x1a
    
    goroutine 18 gp=0x170ac5d843c0 m=nil [GC sweep wait]:
    runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cda788 sp=0x170ac5cda768 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.bgsweep(0x170ac5d92000)
    	/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x170ac5cda7c8 sp=0x170ac5cda788 pc=0x1ee9f11
    runtime.gcenable.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x170ac5cda7e0 sp=0x170ac5cda7c8 pc=0x1edadd7
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cda7e8 sp=0x170ac5cda7e0 pc=0x1f40801
    created by runtime.gcenable in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:214 +0x66
    
    goroutine 19 gp=0x170ac5d845a0 m=nil [GC scavenge wait]:
    runtime.gopark(0x10000?, 0x5981cd8?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdaf78 sp=0x170ac5cdaf58 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.(*scavengerState).park(0x82979c0)
    	/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x170ac5cdafa8 sp=0x170ac5cdaf78 pc=0x1ee77e9
    runtime.bgscavenge(0x170ac5d92000)
    	/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x170ac5cdafc8 sp=0x170ac5cdafa8 pc=0x1ee7d99
    runtime.gcenable.gowrap2()
    	/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x170ac5cdafe0 sp=0x170ac5cdafc8 pc=0x1edad97
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdafe8 sp=0x170ac5cdafe0 pc=0x1f40801
    created by runtime.gcenable in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:215 +0xa5
    
    goroutine 34 gp=0x170ac5e00000 m=nil [GOMAXPROCS updater (idle)]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e06788 sp=0x170ac5e06768 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.updateMaxProcsGoroutine()
    	/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x170ac5e067e0 sp=0x170ac5e06788 pc=0x1f10267
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e067e8 sp=0x170ac5e067e0 pc=0x1f40801
    created by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1
    	/usr/lib/golang/src/runtime/proc.go:7083 +0x37
    
    goroutine 35 gp=0x170ac5e005a0 m=nil [finalizer wait]:
    runtime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x170ac5cde601?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cde620 sp=0x170ac5cde600 pc=0x1f38d2e
    runtime.runFinalizers()
    	/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x170ac5cde7e0 sp=0x170ac5cde620 pc=0x1ed9cc7
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cde7e8 sp=0x170ac5cde7e0 pc=0x1f40801
    created by runtime.createfing in goroutine 1
    	/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d
    
    goroutine 36 gp=0x170ac5ff81e0 m=nil [cleanup wait]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e06f68 sp=0x170ac5e06f48 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.(*cleanupQueue).dequeue(0x82980e0)
    	/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x170ac5e06fa0 sp=0x170ac5e06f68 pc=0x1ed6074
    runtime.runCleanups()
    	/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x170ac5e06fe0 sp=0x170ac5e06fa0 pc=0x1ed66e5
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e06fe8 sp=0x170ac5e06fe0 pc=0x1f40801
    created by runtime.(*cleanupQueue).createGs in goroutine 1
    	/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5
    
    goroutine 37 gp=0x170ac611bc20 m=nil [GC worker (idle)]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e07740 sp=0x170ac5e07720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x170ac5e0b9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5e077c8 sp=0x170ac5e07740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5e077e0 sp=0x170ac5e077c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e077e8 sp=0x170ac5e077e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 3 gp=0x170ac5c1f680 m=nil [runnable]:
    runtime.gopark(0x125c26559f5b?, 0x3?, 0x94?, 0x85?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdf740 sp=0x170ac5cdf720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x170ac5e0b9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdf7c8 sp=0x170ac5cdf740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdf7e0 sp=0x170ac5cdf7c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdf7e8 sp=0x170ac5cdf7e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 20 gp=0x170ac5d84780 m=nil [GC worker (idle)]:
    runtime.gopark(0x125c265c3db1?, 0x3?, 0x92?, 0xcd?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdb740 sp=0x170ac5cdb720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x170ac5e0b9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdb7c8 sp=0x170ac5cdb740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdb7e0 sp=0x170ac5cdb7c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdb7e8 sp=0x170ac5cdb7e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 38 gp=0x170ac6134000 m=nil [GC worker (idle)]:
    runtime.gopark(0x125c26588b9e?, 0x1?, 0xf7?, 0xb4?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e07f40 sp=0x170ac5e07f20 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x170ac5e0b9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5e07fc8 sp=0x170ac5e07f40 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5e07fe0 sp=0x170ac5e07fc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e07fe8 sp=0x170ac5e07fe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 4 gp=0x170ac5c1f860 m=10 mp=0x170ac6280008 [GC mark termination]:
    runtime.systemstack_switch()
    	/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x170ac5cdfea0 sp=0x170ac5cdfe90 pc=0x1f3ebc8
    runtime.forEachP(...)
    	/usr/lib/golang/src/runtime/proc.go:2112
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x170ac5cdff40 sp=0x170ac5cdfea0 pc=0x1edbc34
    runtime.gcBgMarkWorker(0x170ac5e0b9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x170ac5cdffc8 sp=0x170ac5cdff40 pc=0x1edda88
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdffe0 sp=0x170ac5cdffc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdffe8 sp=0x170ac5cdffe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 21 gp=0x170ac5d84960 m=nil [GC worker (idle)]:
    runtime.gopark(0x125c265818cf?, 0x3?, 0xa5?, 0x9c?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdbf40 sp=0x170ac5cdbf20 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x170ac5e0b9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdbfc8 sp=0x170ac5cdbf40 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdbfe0 sp=0x170ac5cdbfc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdbfe8 sp=0x170ac5cdbfe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 22 gp=0x170ac5d84b40 m=nil [GC worker (idle)]:
    runtime.gopark(0x125c265846e7?, 0x3?, 0x77?, 0xb6?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdc740 sp=0x170ac5cdc720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x170ac5e0b9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdc7c8 sp=0x170ac5cdc740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdc7e0 sp=0x170ac5cdc7c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdc7e8 sp=0x170ac5cdc7e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 23 gp=0x170ac5d84d20 m=nil [GC worker (idle)]:
    runtime.gopark(0x125c26581951?, 0x3?, 0x91?, 0x58?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdcf40 sp=0x170ac5cdcf20 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x170ac5e0b9d0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdcfc8 sp=0x170ac5cdcf40 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdcfe0 sp=0x170ac5cdcfc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdcfe8 sp=0x170ac5cdcfe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 39 gp=0x170ac648a3c0 m=nil [select, locked to thread]:
    runtime.gopark(0x170ac5e097a8?, 0x2?, 0x1?, 0xa3?, 0x170ac5e09794?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e09628 sp=0x170ac5e09608 pc=0x1f38d2e
    runtime.selectgo(0x170ac5e097a8, 0x170ac5e09790, 0x0?, 0x0, 0x0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5e09758 sp=0x170ac5e09628 pc=0x1f152a5
    runtime.ensureSigM.func1()
    	/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x170ac5e097e0 sp=0x170ac5e09758 pc=0x1f328e8
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e097e8 sp=0x170ac5e097e0 pc=0x1f40801
    created by runtime.ensureSigM in goroutine 1
    	/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5
    
    goroutine 24 gp=0x170ac62f4960 m=8 mp=0x170ac6180008 [syscall]:
    runtime.notetsleepg(0x82bdea0, 0xffffffffffffffff)
    	/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x170ac5e037a0 sp=0x170ac5e03778 pc=0x1ece509
    os/signal.signal_recv()
    	/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x170ac5e037c0 sp=0x170ac5e037a0 pc=0x1f3b298
    os/signal.loop()
    	/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x170ac5e037e0 sp=0x170ac5e037c0 pc=0x2030553
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e037e8 sp=0x170ac5e037e0 pc=0x1f40801
    created by os/signal.Notify.func1.1 in goroutine 1
    	/usr/lib/golang/src/os/signal/signal.go:152 +0x1f
    
    goroutine 50 gp=0x170ac6135a40 m=nil [select]:
    runtime.gopark(0x170ac66e3798?, 0x2?, 0x80?, 0x6f?, 0x170ac66e378c?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac66e3628 sp=0x170ac66e3608 pc=0x1f38d2e
    runtime.selectgo(0x170ac66e3798, 0x170ac66e3788, 0x0?, 0x0, 0x0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac66e3758 sp=0x170ac66e3628 pc=0x1f152a5
    os/signal.NotifyContext.func1()
    	/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x170ac66e37e0 sp=0x170ac66e3758 pc=0x202ff9c
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac66e37e8 sp=0x170ac66e37e0 pc=0x1f40801
    created by os/signal.NotifyContext in goroutine 1
    	/usr/lib/golang/src/os/signal/signal.go:291 +0x158
    
    goroutine 102 gp=0x170ac61843c0 m=nil [select]:
    runtime.gopark(0x170ac5cefc70?, 0x2?, 0xc8?, 0xf9?, 0x170ac5cefb6c?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cef9f0 sp=0x170ac5cef9d0 pc=0x1f38d2e
    runtime.selectgo(0x170ac5cefc70, 0x170ac5cefb68, 0x554857c?, 0x0, 0x226?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5cefb20 sp=0x170ac5cef9f0 pc=0x1f152a5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x170ac5c49130}, 0x170ac65fa070, 0x170ac6410cf0)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:283 +0x21b fp=0x170ac5ceff48 sp=0x170ac5cefb20 pc=0x39204fb
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x170ac5ceff78 sp=0x170ac5ceff48 pc=0x392800c
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5ceffe0 sp=0x170ac5ceff78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5ceffe8 sp=0x170ac5ceffe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 104 gp=0x170ac6184d20 m=nil [select]:
    runtime.gopark(0x170ac6992ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69929d8?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6992848 sp=0x170ac6992828 pc=0x1f38d2e
    runtime.selectgo(0x170ac6992ac8, 0x170ac69929d0, 0x55562e1?, 0x0, 0x1f6d6cf?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6992978 sp=0x170ac6992848 pc=0x1f152a5
    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac619fb80, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac6992b40 sp=0x170ac6992978 pc=0x22d92e5
    net/http.(*http2ClientConn).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8603
    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac619fb80, {0x4c?, 0xfa?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac6992c80 sp=0x170ac6992b40 pc=0x22d50ad
    net/http.(*http2Transport).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:7852
    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac619fb80?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac6992cc0 sp=0x170ac6992c80 pc=0x22e3a9c
    net/http.(*Transport).roundTrip(0x8175400, 0x170ac619fa40)
    	/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac6992ea0 sp=0x170ac6992cc0 pc=0x231037f
    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)
    	/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac6992ec0 sp=0x170ac6992ea0 pc=0x2321178
    net/http.send(0x170ac619fa40, {0x59a1ce0, 0x8175400}, {0x170ac6993138?, 0x1f3bac6?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69930b0 sp=0x170ac6992ec0 pc=0x22ad9ab
    net/http.(*Client).send(0x170ac6410c60, 0x170ac619fa40, {0x170ac69931e8?, 0x1fdd787?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac6993148 sp=0x170ac69930b0 pc=0x22ad1d8
    net/http.(*Client).do(0x170ac6410c60, 0x170ac619fa40)
    	/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac6993338 sp=0x170ac6993148 pc=0x22af817
    net/http.(*Client).Do(...)
    	/usr/lib/golang/src/net/http/client.go:592
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5c49450}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e34a0}, 0x170ac65f7ce0, {0x170ac65d9560, 0x112})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac6993630 sp=0x170ac6993338 pc=0x38de6bd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5c49450}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x170ac65f7c80, 0x0, {0x0, ...}, ...})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac6993888 sp=0x170ac6993630 pc=0x38ddecd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5c49450?}, 0xa8?, {0x170ac5db1a50?, 0x170ac6764a48?}, 0x3?, 0x2022a4d?)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac6993960 sp=0x170ac6993888 pc=0x38dd40b
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194700?}, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x170ac6764a48, 0x3, 0x4})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69939e0 sp=0x170ac6993960 pc=0x38e51e2
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194700}, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x0?, 0x2?, 0x1?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac6993a78 sp=0x170ac69939e0 pc=0x38e4eaf
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020500}, 0x170ac65fa000)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac6993d60 sp=0x170ac6993a78 pc=0x391b5c5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac5fe1880?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020500?}, 0x0?)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac6993e88 sp=0x170ac6993d60 pc=0x3929c16
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac6993f78 sp=0x170ac6993e88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac6993fe0 sp=0x170ac6993f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6993fe8 sp=0x170ac6993fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 101 gp=0x170ac61d4000 m=nil [sync.WaitGroup.Wait]:
    runtime.gopark(0x170ac5de7e00?, 0x1ed0765?, 0xb0?, 0xfa?, 0xe?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de7da0 sp=0x170ac5de7d80 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x170ac65f8290, 0x0, 0x1, 0x0, 0x19)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x170ac5de7e08 sp=0x170ac5de7da0 pc=0x1f162b2
    sync.runtime_SemacquireWaitGroup(0x170ac6709a70?, 0x0?)
    	/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x170ac5de7e40 sp=0x170ac5de7e08 pc=0x1f3a66e
    sync.(*WaitGroup).Wait(0x170ac65f8288)
    	/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x170ac5de7e68 sp=0x170ac5de7e40 pc=0x1f4f445
    golang.org/x/sync/errgroup.(*Group).Wait(0x170ac65f8280)
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x170ac5de7e88 sp=0x170ac5de7e68 pc=0x362507e
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x170ac5fe1140, {0x59f0b40, 0x170ac5c49130}, {0x170ac626a1e0, 0x6, 0x170ac5cac1c0?}, 0x170ac65fa000)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x170ac5de7f00 sp=0x170ac5de7e88 pc=0x392976a
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x170ac5de7f78 sp=0x170ac5de7f00 pc=0x392808e
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5de7fe0 sp=0x170ac5de7f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de7fe8 sp=0x170ac5de7fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 67 gp=0x170ac62f5a40 m=nil [IO wait]:
    runtime.gopark(0x170ac6497780?, 0x8?, 0x40?, 0x16?, 0xb?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de47b8 sp=0x170ac5de4798 pc=0x1f38d2e
    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)
    	/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x170ac5de47f0 sp=0x170ac5de47b8 pc=0x1ef9e57
    internal/poll.runtime_pollWait(0x7f7eb1670000, 0x72)
    	/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x170ac5de4810 sp=0x170ac5de47f0 pc=0x1f375c5
    internal/poll.(*pollDesc).wait(0x170ac620b480?, 0x170ac65a2000?, 0x0)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x170ac5de4838 sp=0x170ac5de4810 pc=0x1fbd167
    internal/poll.(*pollDesc).waitRead(...)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89
    internal/poll.(*FD).Read(0x170ac620b480, {0x170ac65a2000, 0x3500, 0x3500})
    	/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x170ac5de48d0 sp=0x170ac5de4838 pc=0x1fbe4ce
    net.(*netFD).Read(0x170ac620b480, {0x170ac65a2000?, 0x170ac5de4901?, 0x22a2e75?})
    	/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x170ac5de4918 sp=0x170ac5de48d0 pc=0x20798c5
    net.(*conn).Read(0x170ac62823d8, {0x170ac65a2000?, 0x7f7eb047fc00?, 0x7f7ef9af2e00?})
    	/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x170ac5de4960 sp=0x170ac5de4918 pc=0x208b7a5
    crypto/tls.(*atLeastReader).Read(0x170ac6708f30, {0x170ac65a2000?, 0x170ac5de4a48?, 0x1efcc28?})
    	/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x170ac5de49a8 sp=0x170ac5de4960 pc=0x2222ddb
    bytes.(*Buffer).ReadFrom(0x170ac605fb28, {0x59a3680, 0x170ac6708f30})
    	/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x170ac5de4a00 sp=0x170ac5de49a8 pc=0x1fe9258
    crypto/tls.(*Conn).readFromUntil(0x170ac605f888, {0x59a32c0, 0x170ac62823d8}, 0x222764c?)
    	/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x170ac5de4a38 sp=0x170ac5de4a00 pc=0x2222fbe
    crypto/tls.(*Conn).readRecordOrCCS(0x170ac605f888, 0x0)
    	/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x170ac5de4c18 sp=0x170ac5de4a38 pc=0x222009b
    crypto/tls.(*Conn).readRecord(...)
    	/usr/lib/golang/src/crypto/tls/conn.go:588
    crypto/tls.(*Conn).Read(0x170ac605f888, {0x170ac6029000, 0x1000, 0x22c34e0?})
    	/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x170ac5de4c80 sp=0x170ac5de4c18 pc=0x22267c5
    bufio.(*Reader).Read(0x170ac647aa20, {0x170ac6153624, 0x9, 0x22e100e?})
    	/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0x170ac5de4cb8 sp=0x170ac5de4c80 pc=0x20a71f7
    io.ReadAtLeast({0x59a1260, 0x170ac647aa20}, {0x170ac6153624, 0x9, 0x9}, 0x9)
    	/usr/lib/golang/src/io/io.go:335 +0x8e fp=0x170ac5de4d00 sp=0x170ac5de4cb8 pc=0x1fb17ee
    io.ReadFull(...)
    	/usr/lib/golang/src/io/io.go:354
    net/http.http2readFrameHeader({0x170ac6153624, 0x9, 0x170a00000005?}, {0x59a1260?, 0x170ac647aa20?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:1805 +0x65 fp=0x170ac5de4d50 sp=0x170ac5de4d00 pc=0x22b9e25
    net/http.(*http2Framer).ReadFrameHeader(0x170ac61535e0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:2071 +0x6b fp=0x170ac5de4dd8 sp=0x170ac5de4d50 pc=0x22ba9eb
    net/http.(*http2Framer).ReadFrame(0x170ac61535e0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:2130 +0x18 fp=0x170ac5de4e18 sp=0x170ac5de4dd8 pc=0x22bb1b8
    net/http.(*http2clientConnReadLoop).run(0x170ac5de4fa8)
    	/usr/lib/golang/src/net/http/h2_bundle.go:9550 +0xca fp=0x170ac5de4f60 sp=0x170ac5de4e18 pc=0x22de14a
    net/http.(*http2ClientConn).readLoop(0x170ac61d9880)
    	/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x170ac5de4fc8 sp=0x170ac5de4f60 pc=0x22dd532
    net/http.(*http2Transport).newClientConn.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x170ac5de4fe0 sp=0x170ac5de4fc8 pc=0x22d7057
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de4fe8 sp=0x170ac5de4fe0 pc=0x1f40801
    created by net/http.(*http2Transport).newClientConn in goroutine 66
    	/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5
    
    goroutine 73 gp=0x170ac648ad20 m=nil [IO wait]:
    runtime.gopark(0x170ac6497e40?, 0x1edb176?, 0x80?, 0xed?, 0xb?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de6808 sp=0x170ac5de67e8 pc=0x1f38d2e
    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)
    	/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x170ac5de6840 sp=0x170ac5de6808 pc=0x1ef9e57
    internal/poll.runtime_pollWait(0x7f7eb166fe00, 0x72)
    	/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x170ac5de6860 sp=0x170ac5de6840 pc=0x1f375c5
    internal/poll.(*pollDesc).wait(0x170ac60c4e80?, 0x170ac5ecd300?, 0x0)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x170ac5de6888 sp=0x170ac5de6860 pc=0x1fbd167
    internal/poll.(*pollDesc).waitRead(...)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89
    internal/poll.(*FD).Read(0x170ac60c4e80, {0x170ac5ecd300, 0x1300, 0x1300})
    	/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x170ac5de6920 sp=0x170ac5de6888 pc=0x1fbe4ce
    net.(*netFD).Read(0x170ac60c4e80, {0x170ac5ecd300?, 0x170ac5ecd30a?, 0x5?})
    	/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x170ac5de6968 sp=0x170ac5de6920 pc=0x20798c5
    net.(*conn).Read(0x170ac5d82278, {0x170ac5ecd300?, 0x7f7eb047fc00?, 0x7f7ef9af2e00?})
    	/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x170ac5de69b0 sp=0x170ac5de6968 pc=0x208b7a5
    crypto/tls.(*atLeastReader).Read(0x170ac67088e8, {0x170ac5ecd300?, 0x170ac5de6a98?, 0x1efcc28?})
    	/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x170ac5de69f8 sp=0x170ac5de69b0 pc=0x2222ddb
    bytes.(*Buffer).ReadFrom(0x170ac5dd02a8, {0x59a3680, 0x170ac67088e8})
    	/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x170ac5de6a50 sp=0x170ac5de69f8 pc=0x1fe9258
    crypto/tls.(*Conn).readFromUntil(0x170ac5dd0008, {0x59a32c0, 0x170ac5d82278}, 0x222764c?)
    	/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x170ac5de6a88 sp=0x170ac5de6a50 pc=0x2222fbe
    crypto/tls.(*Conn).readRecordOrCCS(0x170ac5dd0008, 0x0)
    	/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x170ac5de6c68 sp=0x170ac5de6a88 pc=0x222009b
    crypto/tls.(*Conn).readRecord(...)
    	/usr/lib/golang/src/crypto/tls/conn.go:588
    crypto/tls.(*Conn).Read(0x170ac5dd0008, {0x170ac6564000, 0x1000, 0x4e55540?})
    	/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x170ac5de6cd0 sp=0x170ac5de6c68 pc=0x22267c5
    net/http.(*persistConn).Read(0x170ac5f73180, {0x170ac6564000?, 0x59a0ea0?, 0x81260e0?})
    	/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x170ac5de6d30 sp=0x170ac5de6cd0 pc=0x2319d07
    bufio.(*Reader).fill(0x170ac65537a0)
    	/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x170ac5de6d68 sp=0x170ac5de6d30 pc=0x20a6c63
    bufio.(*Reader).Peek(0x170ac65537a0, 0x1)
    	/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x170ac5de6d80 sp=0x170ac5de6d68 pc=0x20a6df3
    net/http.(*persistConn).readLoop(0x170ac5f73180)
    	/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x170ac5de6fc8 sp=0x170ac5de6d80 pc=0x231a812
    net/http.(*Transport).dialConn.gowrap2()
    	/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x170ac5de6fe0 sp=0x170ac5de6fc8 pc=0x2319197
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de6fe8 sp=0x170ac5de6fe0 pc=0x1f40801
    created by net/http.(*Transport).dialConn in goroutine 82
    	/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac
    
    goroutine 103 gp=0x170ac648af00 m=nil [select]:
    runtime.gopark(0x170ac5de0d00?, 0x2?, 0xa0?, 0x6a?, 0x170ac5de0c74?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de0ac8 sp=0x170ac5de0aa8 pc=0x1f38d2e
    runtime.selectgo(0x170ac5de0d00, 0x170ac5de0c70, 0x553bfdc?, 0x0, 0x5a48c80?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5de0bf8 sp=0x170ac5de0ac8 pc=0x1f152a5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x0?, {0x59f0b40, 0x170ac5c49130}, 0x170ac65fa000, 0x170ac65fa070)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x170ac5de0f10 sp=0x170ac5de0bf8 pc=0x392836e
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x170ac5de0f78 sp=0x170ac5de0f10 pc=0x3927f3d
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5de0fe0 sp=0x170ac5de0f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de0fe8 sp=0x170ac5de0fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 74 gp=0x170ac648b4a0 m=nil [select]:
    runtime.gopark(0x170ac5df1f48?, 0x2?, 0x0?, 0xb6?, 0x170ac5df1ef4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6760d88 sp=0x170ac6760d68 pc=0x1f38d2e
    runtime.selectgo(0x170ac6760f48, 0x170ac5df1ef0, 0x170ac68d6980?, 0x0, 0x170ac655e8d0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6760eb8 sp=0x170ac6760d88 pc=0x1f152a5
    net/http.(*persistConn).writeLoop(0x170ac5f73180)
    	/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x170ac6760fc8 sp=0x170ac6760eb8 pc=0x231c426
    net/http.(*Transport).dialConn.gowrap3()
    	/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x170ac6760fe0 sp=0x170ac6760fc8 pc=0x2319157
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6760fe8 sp=0x170ac6760fe0 pc=0x1f40801
    created by net/http.(*Transport).dialConn in goroutine 82
    	/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe
    
    goroutine 105 gp=0x170ac61d43c0 m=nil [select]:
    runtime.gopark(0x170ac699cac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac699c9d8?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac699c848 sp=0x170ac699c828 pc=0x1f38d2e
    runtime.selectgo(0x170ac699cac8, 0x170ac699c9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac699c978 sp=0x170ac699c848 pc=0x1f152a5
    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e000, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac699cb40 sp=0x170ac699c978 pc=0x22d92e5
    net/http.(*http2ClientConn).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8603
    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e000, {0x4c?, 0xfa?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac699cc80 sp=0x170ac699cb40 pc=0x22d50ad
    net/http.(*http2Transport).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:7852
    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e000?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac699ccc0 sp=0x170ac699cc80 pc=0x22e3a9c
    net/http.(*Transport).roundTrip(0x8175400, 0x170ac619fcc0)
    	/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac699cea0 sp=0x170ac699ccc0 pc=0x231037f
    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)
    	/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac699cec0 sp=0x170ac699cea0 pc=0x2321178
    net/http.send(0x170ac619fcc0, {0x59a1ce0, 0x8175400}, {0x170ac699d138?, 0x1f3bac6?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac699d0b0 sp=0x170ac699cec0 pc=0x22ad9ab
    net/http.(*Client).send(0x170ac6410c60, 0x170ac619fcc0, {0x170ac699d1e8?, 0x1fdd787?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac699d148 sp=0x170ac699d0b0 pc=0x22ad1d8
    net/http.(*Client).do(0x170ac6410c60, 0x170ac619fcc0)
    	/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac699d338 sp=0x170ac699d148 pc=0x22af817
    net/http.(*Client).Do(...)
    	/usr/lib/golang/src/net/http/client.go:592
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e000}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e36b0}, 0x170ac65f7e00, {0x170ac65d9680, 0x118})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac699d630 sp=0x170ac699d338 pc=0x38de6bd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e000}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x170ac65f7da0, 0x0, {0x0, ...}, ...})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac699d888 sp=0x170ac699d630 pc=0x38ddecd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e000?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de1a48?}, 0x3?, 0x2022a4d?)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac699d960 sp=0x170ac699d888 pc=0x38dd40b
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194850?}, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x170ac5de1a48, 0x3, 0x4})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac699d9e0 sp=0x170ac699d960 pc=0x38e51e2
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194850}, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x0?, 0x2?, 0x0?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac699da78 sp=0x170ac699d9e0 pc=0x38e4eaf
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac60207c0}, 0x170ac65fa000)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac699dd60 sp=0x170ac699da78 pc=0x391b5c5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac60207c0?}, 0x0?)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac699de88 sp=0x170ac699dd60 pc=0x3929c16
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac699df78 sp=0x170ac699de88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac699dfe0 sp=0x170ac699df78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac699dfe8 sp=0x170ac699dfe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 107 gp=0x170ac61d45a0 m=nil [select]:
    runtime.gopark(0x170ac69b2ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69b29d8?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69b2848 sp=0x170ac69b2828 pc=0x1f38d2e
    runtime.selectgo(0x170ac69b2ac8, 0x170ac69b29d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69b2978 sp=0x170ac69b2848 pc=0x1f152a5
    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e500, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69b2b40 sp=0x170ac69b2978 pc=0x22d92e5
    net/http.(*http2ClientConn).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8603
    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e500, {0x4c?, 0xfa?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69b2c80 sp=0x170ac69b2b40 pc=0x22d50ad
    net/http.(*http2Transport).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:7852
    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e500?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69b2cc0 sp=0x170ac69b2c80 pc=0x22e3a9c
    net/http.(*Transport).roundTrip(0x8175400, 0x170ac699e3c0)
    	/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69b2ea0 sp=0x170ac69b2cc0 pc=0x231037f
    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)
    	/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69b2ec0 sp=0x170ac69b2ea0 pc=0x2321178
    net/http.send(0x170ac699e3c0, {0x59a1ce0, 0x8175400}, {0x170ac69b3138?, 0x1f3bac6?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69b30b0 sp=0x170ac69b2ec0 pc=0x22ad9ab
    net/http.(*Client).send(0x170ac6410c60, 0x170ac699e3c0, {0x4d3aa8?, 0xcd6610?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69b3148 sp=0x170ac69b30b0 pc=0x22ad1d8
    net/http.(*Client).do(0x170ac6410c60, 0x170ac699e3c0)
    	/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69b3338 sp=0x170ac69b3148 pc=0x22af817
    net/http.(*Client).Do(...)
    	/usr/lib/golang/src/net/http/client.go:592
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e1e0}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3a60}, 0x170ac69a0120, {0x170ac65d98c0, 0x111})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69b3630 sp=0x170ac69b3338 pc=0x38de6bd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e1e0}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x170ac69a00c0, 0x0, {0x0, ...}, ...})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69b3888 sp=0x170ac69b3630 pc=0x38ddecd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e1e0?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de2a48?}, 0x3?, 0x2022a4d?)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69b3960 sp=0x170ac69b3888 pc=0x38dd40b
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194bd0?}, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x170ac5de2a48, 0x3, 0x4})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69b39e0 sp=0x170ac69b3960 pc=0x38e51e2
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194bd0}, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x0?, 0x2?, 0x0?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69b3a78 sp=0x170ac69b39e0 pc=0x38e4eaf
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020ec0}, 0x170ac65fa000)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69b3d60 sp=0x170ac69b3a78 pc=0x391b5c5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020ec0?}, 0x0?)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69b3e88 sp=0x170ac69b3d60 pc=0x3929c16
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69b3f78 sp=0x170ac69b3e88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69b3fe0 sp=0x170ac69b3f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69b3fe8 sp=0x170ac69b3fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 106 gp=0x170ac648b680 m=nil [select]:
    runtime.gopark(0x170ac69aaac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69aa9d8?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69aa848 sp=0x170ac69aa828 pc=0x1f38d2e
    runtime.selectgo(0x170ac69aaac8, 0x170ac69aa9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69aa978 sp=0x170ac69aa848 pc=0x1f152a5
    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e280, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69aab40 sp=0x170ac69aa978 pc=0x22d92e5
    net/http.(*http2ClientConn).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8603
    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e280, {0x4c?, 0xfa?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69aac80 sp=0x170ac69aab40 pc=0x22d50ad
    net/http.(*http2Transport).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:7852
    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e280?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69aacc0 sp=0x170ac69aac80 pc=0x22e3a9c
    net/http.(*Transport).roundTrip(0x8175400, 0x170ac699e140)
    	/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69aaea0 sp=0x170ac69aacc0 pc=0x231037f
    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)
    	/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69aaec0 sp=0x170ac69aaea0 pc=0x2321178
    net/http.send(0x170ac699e140, {0x59a1ce0, 0x8175400}, {0x170ac69ab138?, 0x1f3bac6?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69ab0b0 sp=0x170ac69aaec0 pc=0x22ad9ab
    net/http.(*Client).send(0x170ac6410c60, 0x170ac699e140, {0x170ac69ab1e8?, 0x1fdd787?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69ab148 sp=0x170ac69ab0b0 pc=0x22ad1d8
    net/http.(*Client).do(0x170ac6410c60, 0x170ac699e140)
    	/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69ab338 sp=0x170ac69ab148 pc=0x22af817
    net/http.(*Client).Do(...)
    	/usr/lib/golang/src/net/http/client.go:592
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e0f0}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e38c0}, 0x170ac69a0030, {0x170ac65d97a0, 0x10e})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69ab630 sp=0x170ac69ab338 pc=0x38de6bd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e0f0}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x170ac65f7ec0, 0x0, {0x0, ...}, ...})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69ab888 sp=0x170ac69ab630 pc=0x38ddecd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e0f0?}, 0xa8?, {0x170ac5db1a50?, 0x170ac6766a48?}, 0x3?, 0x2022a4d?)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69ab960 sp=0x170ac69ab888 pc=0x38dd40b
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194a80?}, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x170ac6766a48, 0x3, 0x4})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69ab9e0 sp=0x170ac69ab960 pc=0x38e51e2
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194a80}, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x0?, 0x2?, 0x0?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69aba78 sp=0x170ac69ab9e0 pc=0x38e4eaf
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020cc0}, 0x170ac65fa000)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69abd60 sp=0x170ac69aba78 pc=0x391b5c5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac6766ea0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020cc0?}, 0x170ac5f6f3b0?)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69abe88 sp=0x170ac69abd60 pc=0x3929c16
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69abf78 sp=0x170ac69abe88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69abfe0 sp=0x170ac69abf78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69abfe8 sp=0x170ac69abfe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 108 gp=0x170ac61d4780 m=nil [select]:
    runtime.gopark(0x170ac69c0ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69c09d8?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69c0848 sp=0x170ac69c0828 pc=0x1f38d2e
    runtime.selectgo(0x170ac69c0ac8, 0x170ac69c09d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69c0978 sp=0x170ac69c0848 pc=0x1f152a5
    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e780, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69c0b40 sp=0x170ac69c0978 pc=0x22d92e5
    net/http.(*http2ClientConn).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8603
    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e780, {0x4c?, 0xfa?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69c0c80 sp=0x170ac69c0b40 pc=0x22d50ad
    net/http.(*http2Transport).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:7852
    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e780?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69c0cc0 sp=0x170ac69c0c80 pc=0x22e3a9c
    net/http.(*Transport).roundTrip(0x8175400, 0x170ac699e640)
    	/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69c0ea0 sp=0x170ac69c0cc0 pc=0x231037f
    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)
    	/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69c0ec0 sp=0x170ac69c0ea0 pc=0x2321178
    net/http.send(0x170ac699e640, {0x59a1ce0, 0x8175400}, {0x170ac69c1138?, 0x1f3bac6?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69c10b0 sp=0x170ac69c0ec0 pc=0x22ad9ab
    net/http.(*Client).send(0x170ac6410c60, 0x170ac699e640, {0x82bed80?, 0x6401fdd700?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69c1148 sp=0x170ac69c10b0 pc=0x22ad1d8
    net/http.(*Client).do(0x170ac6410c60, 0x170ac699e640)
    	/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69c1338 sp=0x170ac69c1148 pc=0x22af817
    net/http.(*Client).Do(...)
    	/usr/lib/golang/src/net/http/client.go:592
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e280}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3bf0}, 0x170ac69a0210, {0x170ac65d99e0, 0x110})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69c1630 sp=0x170ac69c1338 pc=0x38de6bd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e280}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x170ac69a01b0, 0x0, {0x0, ...}, ...})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69c1888 sp=0x170ac69c1630 pc=0x38ddecd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e280?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de2a48?}, 0x3?, 0x2022a4d?)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69c1960 sp=0x170ac69c1888 pc=0x38dd40b
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194d90?}, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x170ac5de2a48, 0x3, 0x4})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69c19e0 sp=0x170ac69c1960 pc=0x38e51e2
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194d90}, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x0?, 0x2?, 0x0?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69c1a78 sp=0x170ac69c19e0 pc=0x38e4eaf
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6021d80}, 0x170ac65fa000)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69c1d60 sp=0x170ac69c1a78 pc=0x391b5c5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac66e1d6a?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6021d80?}, 0x0?)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69c1e88 sp=0x170ac69c1d60 pc=0x3929c16
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69c1f78 sp=0x170ac69c1e88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69c1fe0 sp=0x170ac69c1f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69c1fe8 sp=0x170ac69c1fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 109 gp=0x170ac61d4960 m=nil [select]:
    runtime.gopark(0x170ac69caac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69ca9d8?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69ca848 sp=0x170ac69ca828 pc=0x1f38d2e
    runtime.selectgo(0x170ac69caac8, 0x170ac69ca9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69ca978 sp=0x170ac69ca848 pc=0x1f152a5
    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699ea00, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69cab40 sp=0x170ac69ca978 pc=0x22d92e5
    net/http.(*http2ClientConn).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8603
    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699ea00, {0x4c?, 0xfa?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69cac80 sp=0x170ac69cab40 pc=0x22d50ad
    net/http.(*http2Transport).RoundTrip(...)
    	/usr/lib/golang/src/net/http/h2_bundle.go:7852
    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699ea00?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69cacc0 sp=0x170ac69cac80 pc=0x22e3a9c
    net/http.(*Transport).roundTrip(0x8175400, 0x170ac699e8c0)
    	/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69caea0 sp=0x170ac69cacc0 pc=0x231037f
    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)
    	/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69caec0 sp=0x170ac69caea0 pc=0x2321178
    net/http.send(0x170ac699e8c0, {0x59a1ce0, 0x8175400}, {0x170ac69cb138?, 0x1f3bac6?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69cb0b0 sp=0x170ac69caec0 pc=0x22ad9ab
    net/http.(*Client).send(0x170ac6410c60, 0x170ac699e8c0, {0x170ac69cb1e8?, 0x1fdd787?, 0x0?})
    	/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69cb148 sp=0x170ac69cb0b0 pc=0x22ad1d8
    net/http.(*Client).do(0x170ac6410c60, 0x170ac699e8c0)
    	/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69cb338 sp=0x170ac69cb148 pc=0x22af817
    net/http.(*Client).Do(...)
    	/usr/lib/golang/src/net/http/client.go:592
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e410}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3d80}, 0x170ac69a0300, {0x170ac65d9b00, 0x114})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69cb630 sp=0x170ac69cb338 pc=0x38de6bd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e410}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x170ac69a02a0, 0x0, {0x0, ...}, ...})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69cb888 sp=0x170ac69cb630 pc=0x38ddecd
    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e410?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de3a48?}, 0x3?, 0x2022a4d?)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69cb960 sp=0x170ac69cb888 pc=0x38dd40b
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194620?}, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x170ac5de3a48, 0x3, 0x4})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69cb9e0 sp=0x170ac69cb960 pc=0x38e51e2
    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194620}, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x0?, 0x2?, 0x2700000000000000?})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69cba78 sp=0x170ac69cb9e0 pc=0x38e4eaf
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac65f81c0}, 0x170ac65fa000)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69cbd60 sp=0x170ac69cba78 pc=0x391b5c5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac65f81c0?}, 0x0?)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69cbe88 sp=0x170ac69cbd60 pc=0x3929c16
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69cbf78 sp=0x170ac69cbe88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69cbfe0 sp=0x170ac69cbf78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69cbfe8 sp=0x170ac69cbfe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 110 gp=0x170ac61d4b40 m=nil [select]:
    runtime.gopark(0x170ac6927f08?, 0x6?, 0x40?, 0x4b?, 0x170ac6927da4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6927c08 sp=0x170ac6927be8 pc=0x1f38d2e
    runtime.selectgo(0x170ac6927f08, 0x170ac6927d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6927d38 sp=0x170ac6927c08 pc=0x1f152a5
    net/http.(*http2clientStream).writeRequest(0x170ac61a0f00, 0x170ac619fb80, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6927f90 sp=0x170ac6927d38 pc=0x22da6a8
    net/http.(*http2clientStream).doRequest(0x170ac61a0f00, 0x0?, 0x170ac66e2798?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6927fb8 sp=0x170ac6927f90 pc=0x22d99d8
    net/http.(*http2ClientConn).roundTrip.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6927fe0 sp=0x170ac6927fb8 pc=0x22d999f
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6927fe8 sp=0x170ac6927fe0 pc=0x1f40801
    created by net/http.(*http2ClientConn).roundTrip in goroutine 104
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470
    
    goroutine 111 gp=0x170ac61d4d20 m=nil [select]:
    runtime.gopark(0x170ac658df08?, 0x6?, 0x0?, 0x7?, 0x170ac658dda4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac658dc08 sp=0x170ac658dbe8 pc=0x1f38d2e
    runtime.selectgo(0x170ac658df08, 0x170ac658dd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac658dd38 sp=0x170ac658dc08 pc=0x1f152a5
    net/http.(*http2clientStream).writeRequest(0x170ac61a1080, 0x170ac699e000, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac658df90 sp=0x170ac658dd38 pc=0x22da6a8
    net/http.(*http2clientStream).doRequest(0x170ac61a1080, 0x0?, 0x170ac66e2798?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac658dfb8 sp=0x170ac658df90 pc=0x22d99d8
    net/http.(*http2ClientConn).roundTrip.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac658dfe0 sp=0x170ac658dfb8 pc=0x22d999f
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac658dfe8 sp=0x170ac658dfe0 pc=0x1f40801
    created by net/http.(*http2ClientConn).roundTrip in goroutine 105
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470
    
    goroutine 112 gp=0x170ac61d4f00 m=nil [select]:
    runtime.gopark(0x170ac6589f08?, 0x6?, 0x0?, 0x7?, 0x170ac6589da4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6589c08 sp=0x170ac6589be8 pc=0x1f38d2e
    runtime.selectgo(0x170ac6589f08, 0x170ac6589d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6589d38 sp=0x170ac6589c08 pc=0x1f152a5
    net/http.(*http2clientStream).writeRequest(0x170ac61a1200, 0x170ac699e280, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6589f90 sp=0x170ac6589d38 pc=0x22da6a8
    net/http.(*http2clientStream).doRequest(0x170ac61a1200, 0x0?, 0x170ac66e1f98?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6589fb8 sp=0x170ac6589f90 pc=0x22d99d8
    net/http.(*http2ClientConn).roundTrip.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6589fe0 sp=0x170ac6589fb8 pc=0x22d999f
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6589fe8 sp=0x170ac6589fe0 pc=0x1f40801
    created by net/http.(*http2ClientConn).roundTrip in goroutine 106
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470
    
    goroutine 113 gp=0x170ac61d50e0 m=nil [select]:
    runtime.gopark(0x170ac658ff08?, 0x6?, 0x0?, 0x7?, 0x170ac658fda4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac658fc08 sp=0x170ac658fbe8 pc=0x1f38d2e
    runtime.selectgo(0x170ac658ff08, 0x170ac658fd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac658fd38 sp=0x170ac658fc08 pc=0x1f152a5
    net/http.(*http2clientStream).writeRequest(0x170ac61a1380, 0x170ac699e780, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac658ff90 sp=0x170ac658fd38 pc=0x22da6a8
    net/http.(*http2clientStream).doRequest(0x170ac61a1380, 0x0?, 0x170ac66e1f98?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac658ffb8 sp=0x170ac658ff90 pc=0x22d99d8
    net/http.(*http2ClientConn).roundTrip.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac658ffe0 sp=0x170ac658ffb8 pc=0x22d999f
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac658ffe8 sp=0x170ac658ffe0 pc=0x1f40801
    created by net/http.(*http2ClientConn).roundTrip in goroutine 108
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470
    
    goroutine 114 gp=0x170ac61d52c0 m=nil [select]:
    runtime.gopark(0x170ac6923f08?, 0x6?, 0x0?, 0x7?, 0x170ac6923da4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6923c08 sp=0x170ac6923be8 pc=0x1f38d2e
    runtime.selectgo(0x170ac6923f08, 0x170ac6923d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6923d38 sp=0x170ac6923c08 pc=0x1f152a5
    net/http.(*http2clientStream).writeRequest(0x170ac61a1500, 0x170ac699ea00, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6923f90 sp=0x170ac6923d38 pc=0x22da6a8
    net/http.(*http2clientStream).doRequest(0x170ac61a1500, 0x0?, 0x170ac66e1f98?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6923fb8 sp=0x170ac6923f90 pc=0x22d99d8
    net/http.(*http2ClientConn).roundTrip.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6923fe0 sp=0x170ac6923fb8 pc=0x22d999f
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6923fe8 sp=0x170ac6923fe0 pc=0x1f40801
    created by net/http.(*http2ClientConn).roundTrip in goroutine 109
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470
    
    goroutine 115 gp=0x170ac61d54a0 m=nil [select]:
    runtime.gopark(0x170ac6925f08?, 0x6?, 0x38?, 0x5b?, 0x170ac6925da4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6925c08 sp=0x170ac6925be8 pc=0x1f38d2e
    runtime.selectgo(0x170ac6925f08, 0x170ac6925d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6925d38 sp=0x170ac6925c08 pc=0x1f152a5
    net/http.(*http2clientStream).writeRequest(0x170ac61a1680, 0x170ac699e500, 0x0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6925f90 sp=0x170ac6925d38 pc=0x22da6a8
    net/http.(*http2clientStream).doRequest(0x170ac61a1680, 0x0?, 0x170ac66e1f98?)
    	/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6925fb8 sp=0x170ac6925f90 pc=0x22d99d8
    net/http.(*http2ClientConn).roundTrip.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6925fe0 sp=0x170ac6925fb8 pc=0x22d999f
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6925fe8 sp=0x170ac6925fe0 pc=0x1f40801
    created by net/http.(*http2ClientConn).roundTrip in goroutine 107
    	/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470
    
    rax    0x0
    rbx    0x7f7eabfff640
    rcx    0x7f7ef9baa94c
    rdx    0x6
    rdi    0x10a2e
    rsi    0x10a30
    rbp    0x10a30
    rsp    0x7f7eabffe790
    r8     0x7f7eabffe860
    r9     0x7f7ef9cd04e0
    r10    0x8
    r11    0x246
    r12    0x6
    r13    0x7f7eabffeb60
    r14    0x170ac5c1f0e0
    r15    0xffffffffffffffff
    rip    0x7f7ef9baa94c
    rflags 0x246
    cs     0x33
    fs     0x0
    gs     0x0
    
      query-infra-both: command failed: exit status 2
    output: runtime/cgo: pthread_create failed: Resource temporarily unavailable
    SIGABRT: abort
    PC=0x7f5e9c16994c m=2 sigcode=18446744073709551610
    
    goroutine 0 gp=0x280029a48b40 m=2 mp=0x280029abb008 [idle]:
    runtime: g 0 gp=0x280029a48b40: unknown pc 0x7f5e9c16994c
    stack: frame={sp:0x7f5e554a7740, fp:0x0} stack=[0x7f5e54ca81e0,0x7f5e554a7de0)
                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef
    00007f5e554a7640: 00000000 01eb64e5  00000000 00000000  .d..............
                      ^ <internal/runtime/syscall/linux.EpollWait+0x45>
    00007f5e554a7650: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7660: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7670: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7680: 00007f5e 554a76d4  00007f5e 554a7d18  .vJU^....}JU^...
    00007f5e554a7690: 00000000 01efa894  00000000 00000000  ................
                      ^ <runtime.netpoll+0xd4>
    00007f5e554a76a0: 00007f5e 554a76d4  00000000 00000000  .vJU^...........
    00007f5e554a76b0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a76c0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a76d0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a76e0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a76f0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7700: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7710: 00000000 00000000  00007f5e 9c176fa9  .........o..^...
    00007f5e554a7720: 00000000 00000000  00000000 00000011  ................
    00007f5e554a7730: 00000000 00000130  00007f5e 9c16993e  0.......>...^...
    00007f5e554a7740: 00000000 00000120  00000000 00000000   ...............
                      ^ SP
    00007f5e554a7750: 00000000 00000013  00000000 00000000  ................
    00007f5e554a7760: 00000000 00000000  00007f5e 9c1eccff  ............^...
    00007f5e554a7770: 00007f5e 327fc910  00007f5e 9c1eccb1  ...2^.......^...
    00007f5e554a7780: 00007f5e 327fc640  00007f5e 554a7a50  @..2^...PzJU^...
    00007f5e554a7790: 00007f5e 554a78be  00007f5e 554a78bf  .xJU^....xJU^...
    00007f5e554a77a0: 00007f5e 327fc640  00007f5e 9c1677f5  @..2^....w..^...
    00007f5e554a77b0: 00007f5e 50000020  00000000 00000120   ..P^... .......
    00007f5e554a77c0: 00000000 003d0f00  5a23f32c fe4c8400  ..=.......L.,.#Z
    00007f5e554a77d0: 00007f5e 554a8640  00000000 00000006  @.JU^...........
    00007f5e554a77e0: 00000000 0bac42a0  00007f5e 554a7b10  .B.......{JU^...
    00007f5e554a77f0: 00002800 29a48b40  00007f5e 9c11c646  @..).(..F...^...
    00007f5e554a7800: 00007f5e 9c2d9e90  00007f5e 9c1067f3  ..-.^....g..^...
    00007f5e554a7810: 00000000 00000020  5a23f32c fe4c8400   .........L.,.#Z
    00007f5e554a7820: 00007f5e 31ffc000  00007f5e 9c1dbac4  ...1^.......^...
    00007f5e554a7830: 00000000 00000000  00000000 0000000d  ................
    runtime: g 0 gp=0x280029a48b40: unknown pc 0x7f5e9c16994c
    stack: frame={sp:0x7f5e554a7740, fp:0x0} stack=[0x7f5e54ca81e0,0x7f5e554a7de0)
                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef
    00007f5e554a7640: 00000000 01eb64e5  00000000 00000000  .d..............
                      ^ <internal/runtime/syscall/linux.EpollWait+0x45>
    00007f5e554a7650: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7660: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7670: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7680: 00007f5e 554a76d4  00007f5e 554a7d18  .vJU^....}JU^...
    00007f5e554a7690: 00000000 01efa894  00000000 00000000  ................
                      ^ <runtime.netpoll+0xd4>
    00007f5e554a76a0: 00007f5e 554a76d4  00000000 00000000  .vJU^...........
    00007f5e554a76b0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a76c0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a76d0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a76e0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a76f0: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7700: 00000000 00000000  00000000 00000000  ................
    00007f5e554a7710: 00000000 00000000  00007f5e 9c176fa9  .........o..^...
    00007f5e554a7720: 00000000 00000000  00000000 00000011  ................
    00007f5e554a7730: 00000000 00000130  00007f5e 9c16993e  0.......>...^...
    00007f5e554a7740: 00000000 00000120  00000000 00000000   ...............
                      ^ SP
    00007f5e554a7750: 00000000 00000013  00000000 00000000  ................
    00007f5e554a7760: 00000000 00000000  00007f5e 9c1eccff  ............^...
    00007f5e554a7770: 00007f5e 327fc910  00007f5e 9c1eccb1  ...2^.......^...
    00007f5e554a7780: 00007f5e 327fc640  00007f5e 554a7a50  @..2^...PzJU^...
    00007f5e554a7790: 00007f5e 554a78be  00007f5e 554a78bf  .xJU^....xJU^...
    00007f5e554a77a0: 00007f5e 327fc640  00007f5e 9c1677f5  @..2^....w..^...
    00007f5e554a77b0: 00007f5e 50000020  00000000 00000120   ..P^... .......
    00007f5e554a77c0: 00000000 003d0f00  5a23f32c fe4c8400  ..=.......L.,.#Z
    00007f5e554a77d0: 00007f5e 554a8640  00000000 00000006  @.JU^...........
    00007f5e554a77e0: 00000000 0bac42a0  00007f5e 554a7b10  .B.......{JU^...
    00007f5e554a77f0: 00002800 29a48b40  00007f5e 9c11c646  @..).(..F...^...
    00007f5e554a7800: 00007f5e 9c2d9e90  00007f5e 9c1067f3  ..-.^....g..^...
    00007f5e554a7810: 00000000 00000020  5a23f32c fe4c8400   .........L.,.#Z
    00007f5e554a7820: 00007f5e 31ffc000  00007f5e 9c1dbac4  ...1^.......^...
    00007f5e554a7830: 00000000 00000000  00000000 0000000d  ................
    
    goroutine 1 gp=0x280029a481e0 m=nil [sync.WaitGroup.Wait]:
    runtime.gopark(0x28002a5927b8?, 0x2022a4d?, 0x30?, 0xe9?, 0x28002a5927e8?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a5927a0 sp=0x28002a592780 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x280029ae34d0, 0x0, 0x1, 0x0, 0x19)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a592808 sp=0x28002a5927a0 pc=0x1f162b2
    sync.runtime_SemacquireWaitGroup(0x28002a83b578?, 0xe0?)
    	/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x28002a592840 sp=0x28002a592808 pc=0x1f3a66e
    sync.(*WaitGroup).Wait(0x280029ae34c8)
    	/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x28002a592868 sp=0x28002a592840 pc=0x1f4f445
    golang.org/x/sync/errgroup.(*Group).Wait(0x280029ae34c0)
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x28002a592888 sp=0x28002a592868 pc=0x362507e
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0}, {0x28002a977b50, 0x1, 0x1})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x28002a592948 sp=0x28002a592888 pc=0x3927e05
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).gatherInfraLogs(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:587 +0x867 fp=0x28002a592d30 sp=0x28002a592948 pc=0x3927727
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:333 +0x533 fp=0x28002a593908 sp=0x28002a592d30 pc=0x3921bf3
    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedInfraQueryOptions).RunInfra(0x28002a11e1e0, {0x59f0b08, 0x28002a1873b0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:83 +0x71f fp=0x28002a593b30 sp=0x28002a593908 pc=0x393887f
    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawInfraQueryOptions).Run(0x0?, {0x59f0b08, 0x28002a1873b0})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_options.go:109 +0x54 fp=0x28002a593b58 sp=0x28002a593b30 pc=0x3939134
    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryInfraCommand.func1(0x28002a53d508?, {0x280029ca3b00?, 0x4?, 0x553b0e2?})
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:45 +0x28 fp=0x28002a593b80 sp=0x28002a593b58 pc=0x3938128
    github.com/spf13/cobra.(*Command).execute(0x28002a53d508, {0x280029ab8120, 0xc, 0xc})
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x28002a593d28 sp=0x28002a593b80 pc=0x212e614
    github.com/spf13/cobra.(*Command).ExecuteC(0x280029ec9b08)
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x28002a593e18 sp=0x28002a593d28 pc=0x212efa5
    github.com/spf13/cobra.(*Command).Execute(...)
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071
    github.com/spf13/cobra.(*Command).ExecuteContext(...)
    	/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064
    main.main()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x28002a593f48 sp=0x28002a593e18 pc=0x47ddd6a
    runtime.main()
    	/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x28002a593fe0 sp=0x28002a593f48 pc=0x1f01975
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a593fe8 sp=0x28002a593fe0 pc=0x1f40801
    
    goroutine 2 gp=0x280029a48d20 m=nil [force gc (idle)]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b08fa8 sp=0x280029b08f88 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.forcegchelper()
    	/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x280029b08fe0 sp=0x280029b08fa8 pc=0x1f01c93
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b08fe8 sp=0x280029b08fe0 pc=0x1f40801
    created by runtime.init.7 in goroutine 1
    	/usr/lib/golang/src/runtime/proc.go:363 +0x1a
    
    goroutine 3 gp=0x280029a492c0 m=nil [GC sweep wait]:
    runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b09788 sp=0x280029b09768 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.bgsweep(0x280029b30000)
    	/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x280029b097c8 sp=0x280029b09788 pc=0x1ee9f11
    runtime.gcenable.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x280029b097e0 sp=0x280029b097c8 pc=0x1edadd7
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b097e8 sp=0x280029b097e0 pc=0x1f40801
    created by runtime.gcenable in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:214 +0x66
    
    goroutine 4 gp=0x280029a494a0 m=nil [GC scavenge wait]:
    runtime.gopark(0x10000?, 0x5981cd8?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b09f78 sp=0x280029b09f58 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.(*scavengerState).park(0x82979c0)
    	/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x280029b09fa8 sp=0x280029b09f78 pc=0x1ee77e9
    runtime.bgscavenge(0x280029b30000)
    	/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x280029b09fc8 sp=0x280029b09fa8 pc=0x1ee7d99
    runtime.gcenable.gowrap2()
    	/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x280029b09fe0 sp=0x280029b09fc8 pc=0x1edad97
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b09fe8 sp=0x280029b09fe0 pc=0x1f40801
    created by runtime.gcenable in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:215 +0xa5
    
    goroutine 5 gp=0x280029a49a40 m=nil [GOMAXPROCS updater (idle)]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0a788 sp=0x280029b0a768 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.updateMaxProcsGoroutine()
    	/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x280029b0a7e0 sp=0x280029b0a788 pc=0x1f10267
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0a7e8 sp=0x280029b0a7e0 pc=0x1f40801
    created by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1
    	/usr/lib/golang/src/runtime/proc.go:7083 +0x37
    
    goroutine 6 gp=0x280029b481e0 m=nil [finalizer wait]:
    runtime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x280029b08601?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b08620 sp=0x280029b08600 pc=0x1f38d2e
    runtime.runFinalizers()
    	/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x280029b087e0 sp=0x280029b08620 pc=0x1ed9cc7
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b087e8 sp=0x280029b087e0 pc=0x1f40801
    created by runtime.createfing in goroutine 1
    	/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d
    
    goroutine 7 gp=0x280029d3fc20 m=nil [cleanup wait]:
    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0af68 sp=0x280029b0af48 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.(*cleanupQueue).dequeue(0x82980e0)
    	/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x280029b0afa0 sp=0x280029b0af68 pc=0x1ed6074
    runtime.runCleanups()
    	/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x280029b0afe0 sp=0x280029b0afa0 pc=0x1ed66e5
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0afe8 sp=0x280029b0afe0 pc=0x1f40801
    created by runtime.(*cleanupQueue).createGs in goroutine 1
    	/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5
    
    goroutine 8 gp=0x280029ee6000 m=nil [GC worker (idle)]:
    runtime.gopark(0x125c86ae32d7?, 0x0?, 0x0?, 0x0?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0b740 sp=0x280029b0b720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x280029ad7ab0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x280029b0b7c8 sp=0x280029b0b740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b0b7e0 sp=0x280029b0b7c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0b7e8 sp=0x280029b0b7e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 18 gp=0x280029b843c0 m=nil [GC worker (idle)]:
    runtime.gopark(0x125c662e5759?, 0x3?, 0xc2?, 0x10?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b04740 sp=0x280029b04720 pc=0x1f38d2e
    runtime.gcBgMarkWorker(0x280029ad7ab0)
    	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x280029b047c8 sp=0x280029b04740 pc=0x1edd8ab
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b047e0 sp=0x280029b047c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b047e8 sp=0x280029b047e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 9 gp=0x280029ee61e0 m=nil [semacquire]:
    runtime.gopark(0x6e2b22?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029b0bc96?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0be38 sp=0x280029b0be18 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029b0bea0 sp=0x280029b0be38 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029b0bf40 sp=0x280029b0bea0 pc=0x1edbb12
    runtime.gcBgMarkWorker(0x280029ad7ab0)
    	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029b0bfc8 sp=0x280029b0bf40 pc=0x1edda88
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b0bfe0 sp=0x280029b0bfc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0bfe8 sp=0x280029b0bfe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 10 gp=0x280029ee63c0 m=nil [semacquire]:
    runtime.gopark(0x1ec6f5e?, 0x1f3ec12?, 0x80?, 0x51?, 0x1ec70df?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eec638 sp=0x280029eec618 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eec6a0 sp=0x280029eec638 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eec740 sp=0x280029eec6a0 pc=0x1edbb12
    runtime.gcBgMarkWorker(0x280029ad7ab0)
    	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eec7c8 sp=0x280029eec740 pc=0x1edda88
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eec7e0 sp=0x280029eec7c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eec7e8 sp=0x280029eec7e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 11 gp=0x280029ee65a0 m=nil [semacquire]:
    runtime.gopark(0x54b522?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029eecc96?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eece38 sp=0x280029eece18 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eecea0 sp=0x280029eece38 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eecf40 sp=0x280029eecea0 pc=0x1edbb12
    runtime.gcBgMarkWorker(0x280029ad7ab0)
    	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eecfc8 sp=0x280029eecf40 pc=0x1edda88
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eecfe0 sp=0x280029eecfc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eecfe8 sp=0x280029eecfe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 34 gp=0x280029f04000 m=nil [semacquire]:
    runtime.gopark(0x2d7122?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029ee8496?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029ee8638 sp=0x280029ee8618 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029ee86a0 sp=0x280029ee8638 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029ee8740 sp=0x280029ee86a0 pc=0x1edbb12
    runtime.gcBgMarkWorker(0x280029ad7ab0)
    	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029ee87c8 sp=0x280029ee8740 pc=0x1edda88
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029ee87e0 sp=0x280029ee87c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029ee87e8 sp=0x280029ee87e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 19 gp=0x280029b845a0 m=4 mp=0x280029b34008 [GC mark termination]:
    runtime.systemstack_switch()
    	/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x280029b04ea0 sp=0x280029b04e90 pc=0x1f3ebc8
    runtime.forEachP(...)
    	/usr/lib/golang/src/runtime/proc.go:2112
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x280029b04f40 sp=0x280029b04ea0 pc=0x1edbc34
    runtime.gcBgMarkWorker(0x280029ad7ab0)
    	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029b04fc8 sp=0x280029b04f40 pc=0x1edda88
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b04fe0 sp=0x280029b04fc8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b04fe8 sp=0x280029b04fe0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 12 gp=0x280029ee6780 m=nil [semacquire]:
    runtime.gopark(0x3f9e22?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029eed496?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eed638 sp=0x280029eed618 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eed6a0 sp=0x280029eed638 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eed740 sp=0x280029eed6a0 pc=0x1edbb12
    runtime.gcBgMarkWorker(0x280029ad7ab0)
    	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eed7c8 sp=0x280029eed740 pc=0x1edda88
    runtime.gcBgMarkStartWorkers.gowrap1()
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eed7e0 sp=0x280029eed7c8 pc=0x1edd797
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eed7e8 sp=0x280029eed7e0 pc=0x1f40801
    created by runtime.gcBgMarkStartWorkers in goroutine 1
    	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105
    
    goroutine 35 gp=0x280029ee74a0 m=nil [select, locked to thread]:
    runtime.gopark(0x280029eef7a8?, 0x2?, 0x1?, 0x74?, 0x280029eef794?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eef628 sp=0x280029eef608 pc=0x1f38d2e
    runtime.selectgo(0x280029eef7a8, 0x280029eef790, 0x0?, 0x0, 0x0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x280029eef758 sp=0x280029eef628 pc=0x1f152a5
    runtime.ensureSigM.func1()
    	/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x280029eef7e0 sp=0x280029eef758 pc=0x1f328e8
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eef7e8 sp=0x280029eef7e0 pc=0x1f40801
    created by runtime.ensureSigM in goroutine 1
    	/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5
    
    goroutine 50 gp=0x280029f84960 m=9 mp=0x28002a000008 [syscall]:
    runtime.notetsleepg(0x82bdea0, 0xffffffffffffffff)
    	/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x280029ee9fa0 sp=0x280029ee9f78 pc=0x1ece509
    os/signal.signal_recv()
    	/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x280029ee9fc0 sp=0x280029ee9fa0 pc=0x1f3b298
    os/signal.loop()
    	/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x280029ee9fe0 sp=0x280029ee9fc0 pc=0x2030553
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029ee9fe8 sp=0x280029ee9fe0 pc=0x1f40801
    created by os/signal.Notify.func1.1 in goroutine 1
    	/usr/lib/golang/src/os/signal/signal.go:152 +0x1f
    
    goroutine 51 gp=0x280029f84b40 m=nil [select]:
    runtime.gopark(0x280029eea798?, 0x2?, 0x80?, 0xa3?, 0x280029eea78c?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eea628 sp=0x280029eea608 pc=0x1f38d2e
    runtime.selectgo(0x280029eea798, 0x280029eea788, 0x0?, 0x0, 0x0?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x280029eea758 sp=0x280029eea628 pc=0x1f152a5
    os/signal.NotifyContext.func1()
    	/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x280029eea7e0 sp=0x280029eea758 pc=0x202ff9c
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eea7e8 sp=0x280029eea7e0 pc=0x1f40801
    created by os/signal.NotifyContext in goroutine 1
    	/usr/lib/golang/src/os/signal/signal.go:291 +0x158
    
    goroutine 137 gp=0x28002a60a000 m=nil [sync.WaitGroup.Wait]:
    runtime.gopark(0x28002a603e00?, 0x1ed0765?, 0x80?, 0xea?, 0x7f5e54234a40?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a603da0 sp=0x28002a603d80 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x280029ae3b10, 0x0, 0x1, 0x0, 0x19)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a603e08 sp=0x28002a603da0 pc=0x1f162b2
    sync.runtime_SemacquireWaitGroup(0x28002a83b590?, 0x0?)
    	/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x28002a603e40 sp=0x28002a603e08 pc=0x1f3a66e
    sync.(*WaitGroup).Wait(0x280029ae3b08)
    	/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x28002a603e68 sp=0x28002a603e40 pc=0x1f4f445
    golang.org/x/sync/errgroup.(*Group).Wait(0x280029ae3b00)
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x28002a603e88 sp=0x28002a603e68 pc=0x362507e
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x280029d80e80, {0x59f0b40, 0x280029beca00}, {0x28002a977b50, 0x1, 0x38d788c?}, 0x28002a830c40)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x28002a603f00 sp=0x28002a603e88 pc=0x392976a
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x28002a603f78 sp=0x28002a603f00 pc=0x392808e
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a603fe0 sp=0x28002a603f78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a603fe8 sp=0x28002a603fe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 38 gp=0x28002a60b0e0 m=nil [IO wait]:
    runtime.gopark(0x28002a7c22c0?, 0x280029aee7f8?, 0x40?, 0x77?, 0xb?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b197b8 sp=0x280029b19798 pc=0x1f38d2e
    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)
    	/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x280029b197f0 sp=0x280029b197b8 pc=0x1ef9e57
    internal/poll.runtime_pollWait(0x7f5e554b6c00, 0x72)
    	/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x280029b19810 sp=0x280029b197f0 pc=0x1f375c5
    internal/poll.(*pollDesc).wait(0x280029bf6400?, 0x28002a90c000?, 0x0)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x280029b19838 sp=0x280029b19810 pc=0x1fbd167
    internal/poll.(*pollDesc).waitRead(...)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89
    internal/poll.(*FD).Read(0x280029bf6400, {0x28002a90c000, 0x3500, 0x3500})
    	/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x280029b198d0 sp=0x280029b19838 pc=0x1fbe4ce
    net.(*netFD).Read(0x280029bf6400, {0x28002a90c000?, 0x28002a90c000?, 0x5?})
    	/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x280029b19918 sp=0x280029b198d0 pc=0x20798c5
    net.(*conn).Read(0x280029a46378, {0x28002a90c000?, 0x7f5e554a9828?, 0x7f5e9c0b1e00?})
    	/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x280029b19960 sp=0x280029b19918 pc=0x208b7a5
    crypto/tls.(*atLeastReader).Read(0x28002aa26ba0, {0x28002a90c000?, 0x28002a0d8658?, 0x1efcc28?})
    	/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x280029b199a8 sp=0x280029b19960 pc=0x2222ddb
    bytes.(*Buffer).ReadFrom(0x28002a0d8628, {0x59a3680, 0x28002aa26ba0})
    	/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x280029b19a00 sp=0x280029b199a8 pc=0x1fe9258
    crypto/tls.(*Conn).readFromUntil(0x28002a0d8388, {0x59a32c0, 0x280029a46378}, 0x222764c?)
    	/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x280029b19a38 sp=0x280029b19a00 pc=0x2222fbe
    crypto/tls.(*Conn).readRecordOrCCS(0x28002a0d8388, 0x0)
    	/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x280029b19c18 sp=0x280029b19a38 pc=0x222009b
    crypto/tls.(*Conn).readRecord(...)
    	/usr/lib/golang/src/crypto/tls/conn.go:588
    crypto/tls.(*Conn).Read(0x28002a0d8388, {0x280029fed000, 0x1000, 0x22c34e0?})
    	/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x280029b19c80 sp=0x280029b19c18 pc=0x22267c5
    bufio.(*Reader).Read(0x28002a7b89c0, {0x28002a7c8584, 0x9, 0x22e100e?})
    	/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0x280029b19cb8 sp=0x280029b19c80 pc=0x20a71f7
    io.ReadAtLeast({0x59a1260, 0x28002a7b89c0}, {0x28002a7c8584, 0x9, 0x9}, 0x9)
    	/usr/lib/golang/src/io/io.go:335 +0x8e fp=0x280029b19d00 sp=0x280029b19cb8 pc=0x1fb17ee
    io.ReadFull(...)
    	/usr/lib/golang/src/io/io.go:354
    net/http.http2readFrameHeader({0x28002a7c8584, 0x9, 0x280000000015?}, {0x59a1260?, 0x28002a7b89c0?})
    	/usr/lib/golang/src/net/http/h2_bundle.go:1805 +0x65 fp=0x280029b19d50 sp=0x280029b19d00 pc=0x22b9e25
    net/http.(*http2Framer).ReadFrameHeader(0x28002a7c8540)
    	/usr/lib/golang/src/net/http/h2_bundle.go:2071 +0x6b fp=0x280029b19dd8 sp=0x280029b19d50 pc=0x22ba9eb
    net/http.(*http2Framer).ReadFrame(0x28002a7c8540)
    	/usr/lib/golang/src/net/http/h2_bundle.go:2130 +0x18 fp=0x280029b19e18 sp=0x280029b19dd8 pc=0x22bb1b8
    net/http.(*http2clientConnReadLoop).run(0x280029b19fa8)
    	/usr/lib/golang/src/net/http/h2_bundle.go:9550 +0xca fp=0x280029b19f60 sp=0x280029b19e18 pc=0x22de14a
    net/http.(*http2ClientConn).readLoop(0x280029bc01c0)
    	/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x280029b19fc8 sp=0x280029b19f60 pc=0x22dd532
    net/http.(*http2Transport).newClientConn.gowrap1()
    	/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x280029b19fe0 sp=0x280029b19fc8 pc=0x22d7057
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b19fe8 sp=0x280029b19fe0 pc=0x1f40801
    created by net/http.(*http2Transport).newClientConn in goroutine 37
    	/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5
    
    goroutine 69 gp=0x28002a60b4a0 m=nil [IO wait]:
    runtime.gopark(0x28002a50c560?, 0x1edb176?, 0x80?, 0xed?, 0xb?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a606808 sp=0x28002a6067e8 pc=0x1f38d2e
    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)
    	/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x28002a606840 sp=0x28002a606808 pc=0x1ef9e57
    internal/poll.runtime_pollWait(0x7f5e554b6a00, 0x72)
    	/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x28002a606860 sp=0x28002a606840 pc=0x1f375c5
    internal/poll.(*pollDesc).wait(0x280029bf6c80?, 0x280029a5a000?, 0x0)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x28002a606888 sp=0x28002a606860 pc=0x1fbd167
    internal/poll.(*pollDesc).waitRead(...)
    	/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89
    internal/poll.(*FD).Read(0x280029bf6c80, {0x280029a5a000, 0x2500, 0x2500})
    	/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x28002a606920 sp=0x28002a606888 pc=0x1fbe4ce
    net.(*netFD).Read(0x280029bf6c80, {0x280029a5a000?, 0x28002a60690a?, 0x1edb176?})
    	/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x28002a606968 sp=0x28002a606920 pc=0x20798c5
    net.(*conn).Read(0x280029a463c0, {0x280029a5a000?, 0x7f5e5423eae0?, 0x7f5e9c09dde0?})
    	/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x28002a6069b0 sp=0x28002a606968 pc=0x208b7a5
    crypto/tls.(*atLeastReader).Read(0x28002a628318, {0x280029a5a000?, 0x28002a606a98?, 0x1efcc28?})
    	/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x28002a6069f8 sp=0x28002a6069b0 pc=0x2222ddb
    bytes.(*Buffer).ReadFrom(0x280029ba9ea8, {0x59a3680, 0x28002a628318})
    	/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x28002a606a50 sp=0x28002a6069f8 pc=0x1fe9258
    crypto/tls.(*Conn).readFromUntil(0x280029ba9c08, {0x59a32c0, 0x280029a463c0}, 0x222764c?)
    	/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x28002a606a88 sp=0x28002a606a50 pc=0x2222fbe
    crypto/tls.(*Conn).readRecordOrCCS(0x280029ba9c08, 0x0)
    	/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x28002a606c68 sp=0x28002a606a88 pc=0x222009b
    crypto/tls.(*Conn).readRecord(...)
    	/usr/lib/golang/src/crypto/tls/conn.go:588
    crypto/tls.(*Conn).Read(0x280029ba9c08, {0x28002a24a000, 0x1000, 0x4e55540?})
    	/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x28002a606cd0 sp=0x28002a606c68 pc=0x22267c5
    net/http.(*persistConn).Read(0x28002a0ad400, {0x28002a24a000?, 0x59a0ea0?, 0x81260e0?})
    	/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x28002a606d30 sp=0x28002a606cd0 pc=0x2319d07
    bufio.(*Reader).fill(0x28002a8130e0)
    	/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x28002a606d68 sp=0x28002a606d30 pc=0x20a6c63
    bufio.(*Reader).Peek(0x28002a8130e0, 0x1)
    	/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x28002a606d80 sp=0x28002a606d68 pc=0x20a6df3
    net/http.(*persistConn).readLoop(0x28002a0ad400)
    	/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x28002a606fc8 sp=0x28002a606d80 pc=0x231a812
    net/http.(*Transport).dialConn.gowrap2()
    	/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x28002a606fe0 sp=0x28002a606fc8 pc=0x2319197
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a606fe8 sp=0x28002a606fe0 pc=0x1f40801
    created by net/http.(*Transport).dialConn in goroutine 55
    	/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac
    
    goroutine 70 gp=0x28002a60bc20 m=nil [select]:
    runtime.gopark(0x28002a613f48?, 0x2?, 0x0?, 0xb6?, 0x28002a613ef4?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a708d88 sp=0x28002a708d68 pc=0x1f38d2e
    runtime.selectgo(0x28002a708f48, 0x28002a613ef0, 0x280029c93a80?, 0x0, 0x28002a4f7c80?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a708eb8 sp=0x28002a708d88 pc=0x1f152a5
    net/http.(*persistConn).writeLoop(0x28002a0ad400)
    	/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x28002a708fc8 sp=0x28002a708eb8 pc=0x231c426
    net/http.(*Transport).dialConn.gowrap3()
    	/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x28002a708fe0 sp=0x28002a708fc8 pc=0x2319157
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a708fe8 sp=0x28002a708fe0 pc=0x1f40801
    created by net/http.(*Transport).dialConn in goroutine 55
    	/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe
    
    goroutine 138 gp=0x28002a84a780 m=nil [semacquire]:
    runtime.gopark(0x1?, 0x1f78fa5?, 0xe0?, 0x75?, 0x2?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a2ca910 sp=0x28002a2ca8f0 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a2ca978 sp=0x28002a2ca910 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x28002a2caa18 sp=0x28002a2ca978 pc=0x1edbb12
    runtime.gcAssistAlloc(0x28002a84a780)
    	/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x28002a2caaa8 sp=0x28002a2caa18 pc=0x1ee06b6
    runtime.deductAssistCredit(0x28002a2cac70?)
    	/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x28002a2caac8 sp=0x28002a2caaa8 pc=0x1ed10a6
    runtime.mallocgc(0x10, 0x4af39e0, 0x1)
    	/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x28002a2caaf8 sp=0x28002a2caac8 pc=0x1f35e07
    runtime.convTstring({0x28002a6a7728, 0x14})
    	/usr/lib/golang/src/runtime/iface.go:431 +0x45 fp=0x28002a2cab20 sp=0x28002a2caaf8 pc=0x1f35865
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x280029beca00}, 0x28002a830cb0, 0x280029f636e0)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:297 +0x394 fp=0x28002a2caf48 sp=0x28002a2cab20 pc=0x3920674
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x28002a2caf78 sp=0x28002a2caf48 pc=0x392800c
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a2cafe0 sp=0x28002a2caf78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a2cafe8 sp=0x28002a2cafe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 139 gp=0x28002a84a960 m=nil [select]:
    runtime.gopark(0x28002a4bdd00?, 0x2?, 0x1?, 0xc9?, 0x28002a4bdc74?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a4bdac8 sp=0x28002a4bdaa8 pc=0x1f38d2e
    runtime.selectgo(0x28002a4bdd00, 0x28002a4bdc70, 0x28002a3e8a50?, 0x0, 0x280029f827c8?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a4bdbf8 sp=0x28002a4bdac8 pc=0x1f152a5
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x49424d554554534c?, {0x59f0b40, 0x280029beca00}, 0x28002a830c40, 0x28002a830cb0)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x28002a4bdf10 sp=0x28002a4bdbf8 pc=0x392836e
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x28002a4bdf78 sp=0x28002a4bdf10 pc=0x3927f3d
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a4bdfe0 sp=0x28002a4bdf78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a4bdfe8 sp=0x28002a4bdfe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 140 gp=0x28002a84ab40 m=nil [semacquire]:
    runtime.gopark(0x1f6aa32?, 0x28002a34b418?, 0xc5?, 0xbd?, 0x0?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a34b3c0 sp=0x28002a34b3a0 pc=0x1f38d2e
    runtime.goparkunlock(...)
    	/usr/lib/golang/src/runtime/proc.go:468
    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)
    	/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a34b428 sp=0x28002a34b3c0 pc=0x1f162b2
    runtime.semacquire(...)
    	/usr/lib/golang/src/runtime/sema.go:143
    runtime.gcMarkDone()
    	/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x28002a34b4c8 sp=0x28002a34b428 pc=0x1edbb12
    runtime.gcAssistAlloc(0x28002a84ab40)
    	/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x28002a34b558 sp=0x28002a34b4c8 pc=0x1ee06b6
    runtime.deductAssistCredit(0x20?)
    	/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x28002a34b578 sp=0x28002a34b558 pc=0x1ed10a6
    runtime.mallocgc(0x1000, 0x4af32a0, 0x1)
    	/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x28002a34b5a8 sp=0x28002a34b578 pc=0x1f35e07
    runtime.makeslice(0x28002a34b668?, 0x1f8?, 0x1ff?)
    	/usr/lib/golang/src/runtime/slice.go:117 +0x49 fp=0x28002a34b5d0 sp=0x28002a34b5a8 pc=0x1f3b489
    bufio.NewWriterSize(...)
    	/usr/lib/golang/src/bufio/bufio.go:599
    bufio.NewWriter(...)
    	/usr/lib/golang/src/bufio/bufio.go:608
    encoding/csv.NewWriter(...)
    	/usr/lib/golang/src/encoding/csv/writer.go:42
    github.com/Azure/azure-kusto-go/azkustodata/query.(*row).String(0x812ab90?)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/row_impl.go:100 +0x18e fp=0x28002a34b6e8 sp=0x28002a34b5d0 pc=0x38cf5ee
    fmt.(*pp).handleMethods(0x28002a24c0d0, 0x2a34b8f0?)
    	/usr/lib/golang/src/fmt/print.go:674 +0x2f4 fp=0x28002a34b8a8 sp=0x28002a34b6e8 pc=0x1fdd1d4
    fmt.(*pp).printArg(0x28002a24c0d0, {0x549a3a0, 0x28002a7c7c80}, 0x76)
    	/usr/lib/golang/src/fmt/print.go:757 +0x42e fp=0x28002a34b938 sp=0x28002a34b8a8 pc=0x1fdd9ee
    fmt.(*pp).doPrintf(0x28002a24c0d0, {0x553a400, 0x2}, {0x28002a34bd08, 0x1, 0x1})
    	/usr/lib/golang/src/fmt/print.go:1075 +0x3ec fp=0x28002a34ba20 sp=0x28002a34b938 pc=0x1fe04ec
    fmt.Sprintf({0x553a400, 0x2}, {0x28002a34bd08, 0x1, 0x1})
    	/usr/lib/golang/src/fmt/print.go:239 +0x53 fp=0x28002a34ba78 sp=0x28002a34ba20 pc=0x1fda6b3
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x280029e454e0, {0x59f0b40, 0x280029beca50}, {0x5a00300, 0x280029ae2040}, 0x28002a830c40)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:160 +0x8d8 fp=0x28002a34bd60 sp=0x28002a34ba78 pc=0x391bad8
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x2d20414320746f6f?, {0x59f0b40?, 0x280029beca50?}, {0x5a00300?, 0x280029ae2040?}, 0x43437a4b4349494d?)
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x28002a34be88 sp=0x28002a34bd60 pc=0x3929c16
    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()
    	/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x28002a34bf78 sp=0x28002a34be88 pc=0x392984a
    golang.org/x/sync/errgroup.(*Group).Go.func1()
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a34bfe0 sp=0x28002a34bf78 pc=0x36251d0
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a34bfe8 sp=0x28002a34bfe0 pc=0x1f40801
    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 137
    	/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95
    
    goroutine 142 gp=0x28002a84af00 m=nil [select]:
    runtime.gopark(0x28002a333de0?, 0x2?, 0x8?, 0xb8?, 0x28002a333dbc?)
    	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a333c58 sp=0x28002a333c38 pc=0x1f38d2e
    runtime.selectgo(0x28002a333de0, 0x28002a333db8, 0x280029afc008?, 0x1, 0x1f3665f?, 0x1)
    	/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a333d88 sp=0x28002a333c58 pc=0x1f152a5
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.(*iterativeDataset).sendTable(0x280029becf00, {0x5a19be0, 0x28002ab876c0})
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:378 +0xf3 fp=0x28002a333e10 sp=0x28002a333d88 pc=0x38d8d33
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.handleDataTable(0x280029becf00, 0x28002a3b2b40)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:324 +0x2c6 fp=0x28002a333ec8 sp=0x28002a333e10 pc=0x38d8686
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x280029becf00)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:167 +0x148 fp=0x28002a333f30 sp=0x28002a333ec8 pc=0x38d7b48
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x280029becf00, 0x28002a977da0)
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x28002a333fc0 sp=0x28002a333f30 pc=0x38d788c
    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x28002a333fe0 sp=0x28002a333fc0 pc=0x38d751b
    runtime.goexit({})
    	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a333fe8 sp=0x28002a333fe0 pc=0x1f40801
    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 140
    	/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc
    
    rax    0x0
    rbx    0x7f5e554a8640
    rcx    0x7f5e9c16994c
    rdx    0x6
    rdi    0x10a66
    rsi    0x10a67
    rbp    0x10a67
    rsp    0x7f5e554a7740
    r8     0x7f5e554a7810
    r9     0x7f5e9c28f4e0
    r10    0x8
    r11    0x246
    r12    0x6
    r13    0x7f5e554a7b10
    r14    0x280029a48b40
    r15    0xffffffffffffffff
    rip    0x7f5e9c16994c
    rflags 0x246
    cs     0x33
    fs     0x0
    gs     0x0
    
    {
        s: "must-gather CLI test failures:\n  query-basic: command failed: exit status 1\noutput: \x1b[37m[21:33:14.493]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104\"\n}\x1b[0m\n\x1b[37m[21:33:23.438]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:21.8572874Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\\\", ActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", SubActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \\u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\\\"}, IsPermanent:false})\",\n  \"name\": \"detailedInfraOrchestrationLogs\"\n}\x1b[0m\n\x1b[37m[21:33:23.438]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"query \\\"backendControllerConditions\\\" time window 50/289 failed (KQL: let _startTime = datetime(2026-10-15T01:38:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T01:43:14.4931101Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-15T01:38:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T01:43:14.4931101Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n  \"name\": \"backendControllerConditions\"\n}\x1b[0m\n\x1b[37m[21:33:23.712]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgenerating Cosmos content Git history\x1b[0m \x1b[90m{\n  \"input\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/cosmosContent\"\n}\x1b[0m\n\x1b[37m[21:33:23.712]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mParsing log file\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n}\x1b[0m\n\x1b[37m[21:33:24.208]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mFound data dump entries\x1b[0m \x1b[90m{\n  \"count\": 1498,\n  \"files\": 1\n}\x1b[0m\n\x1b[37m[21:33:24.208]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mInitializing git repository\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/cosmosContent\"\n}\x1b[0m\n\x1b[37m[21:33:24.217]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mProcessing entries and creating commits\x1b[0m \x1b[90m{}\x1b[0m\n\x1b[37m[21:33:40.606]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mCompleted\x1b[0m \x1b[90m{\n  \"commits\": 793\n}\x1b[0m\n\x1b[37m[21:33:40.607]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for clusters in the resource group\x1b[0m \x1b[90m{\n  \"clusterIDs\": null,\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect\"\n}\x1b[0m\n\x1b[37m[21:33:40.961]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: cluster IDs to inspect\x1b[0m \x1b[90m{\n  \"clusterIDs\": [\n    \"2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n  ]\n}\x1b[0m\n\x1b[37m[21:33:41.088]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered infra cluster names\x1b[0m \x1b[90m{\n  \"clusterNames\": [\n    \"ci01-j8700672-mgmt-1\",\n    \"ci01-j8700672-svc\"\n  ]\n}\x1b[0m\n\x1b[37m[21:33:41.127]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespaces\": [\n    \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n  ]\n}\x1b[0m\n\x1b[37m[21:33:41.127]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\x1b[0m \x1b[90m{\n  \"managementCluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespaceCount\": 4\n}\x1b[0m\n\x1b[37m[21:33:41.127]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:33:41.317]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:33:41.478]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:33:41.623]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n}\x1b[0m\n\x1b[37m[21:33:43.044]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mmust-gather finished with errors; partial content may have been written\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:21.8572874Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\\\", ActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", SubActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \\u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\\\"}, IsPermanent:false})\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104\"\n}\x1b[0m\n\x1b[37m[21:33:43.044]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mcommand failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:21.8572874Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\\\", ActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", SubActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \\u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\\\"}, IsPermanent:false})\"\n}\x1b[0m\n\n  query-skip-hcp-logs: command failed: exit status 1\noutput: \x1b[37m[21:33:43.407]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843\"\n}\x1b[0m\n\x1b[37m[21:33:49.648]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:47.6716888Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\\\", ActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", SubActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \\u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\\\"}, IsPermanent:false})\",\n  \"name\": \"detailedInfraOrchestrationLogs\"\n}\x1b[0m\n\x1b[37m[21:33:49.648]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"query \\\"backendControllerConditions\\\" time window 47/289 failed (KQL: let _startTime = datetime(2026-10-15T01:23:43.4070857Z);\\nlet _endTime = datetime(2026-10-15T01:28:43.4070856Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n  \"name\": \"backendControllerConditions\"\n}\x1b[0m\n\x1b[37m[21:33:49.744]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgenerating Cosmos content Git history\x1b[0m \x1b[90m{\n  \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/cosmosContent\"\n}\x1b[0m\n\x1b[37m[21:33:49.744]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mParsing log file\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n}\x1b[0m\n\x1b[37m[21:33:50.271]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mFound data dump entries\x1b[0m \x1b[90m{\n  \"count\": 1498,\n  \"files\": 1\n}\x1b[0m\n\x1b[37m[21:33:50.271]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mInitializing git repository\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/cosmosContent\"\n}\x1b[0m\n\x1b[37m[21:33:50.281]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mProcessing entries and creating commits\x1b[0m \x1b[90m{}\x1b[0m\n\x1b[37m[21:34:08.647]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mCompleted\x1b[0m \x1b[90m{\n  \"commits\": 793\n}\x1b[0m\n\x1b[37m[21:34:08.648]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for clusters in the resource group\x1b[0m \x1b[90m{\n  \"clusterIDs\": null,\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect\"\n}\x1b[0m\n\x1b[37m[21:34:09.212]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: cluster IDs to inspect\x1b[0m \x1b[90m{\n  \"clusterIDs\": [\n    \"2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n  ]\n}\x1b[0m\n\x1b[37m[21:34:09.309]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered infra cluster names\x1b[0m \x1b[90m{\n  \"clusterNames\": [\n    \"ci01-j8700672-mgmt-1\",\n    \"ci01-j8700672-svc\"\n  ]\n}\x1b[0m\n\x1b[37m[21:34:09.470]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespaces\": [\n    \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n  ]\n}\x1b[0m\n\x1b[37m[21:34:09.470]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\x1b[0m \x1b[90m{\n  \"managementCluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespaceCount\": 4\n}\x1b[0m\n\x1b[37m[21:34:09.470]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:34:09.615]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:34:10.183]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:34:10.617]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n}\x1b[0m\n\x1b[37m[21:34:12.156]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mmust-gather finished with errors; partial content may have been written\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:47.6716888Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\\\", ActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", SubActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \\u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\\\"}, IsPermanent:false})\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843\"\n}\x1b[0m\n\x1b[37m[21:34:12.157]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mcommand failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:47.6716888Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\\\", ActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", SubActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \\u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\\\"}, IsPermanent:false})\"\n}\x1b[0m\n\n  query-skip-kubernetes-events: command failed: exit status 1\noutput: \x1b[37m[21:34:12.559]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767\"\n}\x1b[0m\n\x1b[37m[21:34:18.220]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:16.3210435Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\\\", ActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", SubActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \\u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\\\"}, IsPermanent:false})\",\n  \"name\": \"detailedInfraOrchestrationLogs\"\n}\x1b[0m\n\x1b[37m[21:34:18.221]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"query \\\"backendControllerConditions\\\" time window 47/289 failed (KQL: let _startTime = datetime(2026-10-15T01:24:12.5582990Z);\\nlet _endTime = datetime(2026-10-15T01:29:12.5582989Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n  \"name\": \"backendControllerConditions\"\n}\x1b[0m\n\x1b[37m[21:34:18.221]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgenerating Cosmos content Git history\x1b[0m \x1b[90m{\n  \"input\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/cosmosContent\"\n}\x1b[0m\n\x1b[37m[21:34:18.221]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mParsing log file\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n}\x1b[0m\n\x1b[37m[21:34:18.723]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mFound data dump entries\x1b[0m \x1b[90m{\n  \"count\": 1498,\n  \"files\": 1\n}\x1b[0m\n\x1b[37m[21:34:18.724]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mInitializing git repository\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/cosmosContent\"\n}\x1b[0m\n\x1b[37m[21:34:18.734]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mProcessing entries and creating commits\x1b[0m \x1b[90m{}\x1b[0m\n\x1b[37m[21:34:35.437]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for clusters in the resource group\x1b[0m \x1b[90m{\n  \"clusterIDs\": null,\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect\"\n}\x1b[0m\n\x1b[37m[21:34:35.845]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: cluster IDs to inspect\x1b[0m \x1b[90m{\n  \"clusterIDs\": [\n    \"2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n  ]\n}\x1b[0m\n\x1b[37m[21:34:36.194]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered infra cluster names\x1b[0m \x1b[90m{\n  \"clusterNames\": [\n    \"ci01-j8700672-mgmt-1\",\n    \"ci01-j8700672-svc\"\n  ]\n}\x1b[0m\n\x1b[37m[21:34:36.313]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespaces\": [\n    \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n  ]\n}\x1b[0m\n\x1b[37m[21:34:36.313]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\x1b[0m \x1b[90m{\n  \"managementCluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespaceCount\": 4\n}\x1b[0m\n\x1b[37m[21:34:36.313]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:34:36.544]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:34:36.863]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:34:37.093]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n}\x1b[0m\n\x1b[37m[21:34:38.438]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mmust-gather finished with errors; partial content may have been written\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:16.3210435Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\\\", ActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", SubActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \\u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767\"\n}\x1b[0m\n\x1b[37m[21:34:38.438]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mcommand failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:16.3210435Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\\\", ActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", SubActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \\u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n}\x1b[0m\n\n  query-collect-systemd-logs: command failed: exit status 1\noutput: \x1b[37m[21:34:38.772]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904\"\n}\x1b[0m\n\x1b[37m[21:34:46.539]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:46.3046137Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\\\", ActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", SubActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \\u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\\\"}, IsPermanent:false})\",\n  \"name\": \"detailedInfraOrchestrationLogs\"\n}\x1b[0m\n\x1b[37m[21:34:46.539]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"query \\\"backendControllerConditions\\\" time window 45/289 failed (KQL: let _startTime = datetime(2026-10-15T01:14:38.7716298Z);\\nlet _endTime = datetime(2026-10-15T01:19:38.7716297Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n  \"name\": \"backendControllerConditions\"\n}\x1b[0m\n\x1b[37m[21:34:47.129]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgenerating Cosmos content Git history\x1b[0m \x1b[90m{\n  \"input\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/cosmosContent\"\n}\x1b[0m\n\x1b[37m[21:34:47.129]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mParsing log file\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n}\x1b[0m\n\x1b[37m[21:34:47.633]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mFound data dump entries\x1b[0m \x1b[90m{\n  \"count\": 1498,\n  \"files\": 1\n}\x1b[0m\n\x1b[37m[21:34:47.633]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mInitializing git repository\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/cosmosContent\"\n}\x1b[0m\n\x1b[37m[21:34:47.642]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mProcessing entries and creating commits\x1b[0m \x1b[90m{}\x1b[0m\n\x1b[37m[21:34:59.291]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for clusters in the resource group\x1b[0m \x1b[90m{\n  \"clusterIDs\": null,\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect\"\n}\x1b[0m\n\x1b[37m[21:34:59.911]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: cluster IDs to inspect\x1b[0m \x1b[90m{\n  \"clusterIDs\": [\n    \"2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n  ]\n}\x1b[0m\n\x1b[37m[21:35:00.221]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered infra cluster names\x1b[0m \x1b[90m{\n  \"clusterNames\": [\n    \"ci01-j8700672-mgmt-1\",\n    \"ci01-j8700672-svc\"\n  ]\n}\x1b[0m\n\x1b[37m[21:35:00.360]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespaces\": [\n    \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n    \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n  ]\n}\x1b[0m\n\x1b[37m[21:35:00.360]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\x1b[0m \x1b[90m{\n  \"managementCluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespaceCount\": 4\n}\x1b[0m\n\x1b[37m[21:35:00.360]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:35:00.702]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:35:01.111]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n}\x1b[0m\n\x1b[37m[21:35:01.638]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j8700672-mgmt-1\",\n  \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n}\x1b[0m\n\x1b[37m[21:35:02.339]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mmust-gather finished with errors; partial content may have been written\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:46.3046137Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\\\", ActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", SubActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \\u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904\"\n}\x1b[0m\n\x1b[37m[21:35:02.340]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mcommand failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:46.3046137Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\\\", ActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", SubActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \\u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n}\x1b[0m\n\n  query-skip-hcp-and-events: command failed: exit status 2\noutput: \x1b[37m[21:35:02.599]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-1950984811\"\n}\x1b[0m\nruntime/cgo: pthread_create failed: Resource temporarily unavailable\nSIGABRT: abort\nPC=0x7f80df60594c m=4 sigcode=18446744073709551610\n\ngoroutine 0 gp=0x14197f1334a0 m=4 mp=0x14197f21a008 [idle]:\nruntime: g 0 gp=0x14197f1334a0: unknown pc 0x7f80df60594c\nstack: frame={sp:0x7f808bffe790, fp:0x0} stack=[0x7f808b7ff1e0,0x7f808bffede0)\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n00007f808bffe690: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6a0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6b0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6c0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6d0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6e0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6f0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe700: 00000000 00000000  00000000 00000000  ................\n00007f808bffe710: 00000000 00000000  00000000 00000000  ................\n00007f808bffe720: 00000000 00000000  00000000 00000000  ................\n00007f808bffe730: 00000000 00000000  00000000 00000000  ................\n00007f808bffe740: 00000000 00000000  00000000 00000000  ................\n00007f808bffe750: 00000000 00000000  00000000 00000000  ................\n00007f808bffe760: 00000000 00000000  00007f80 df612fa9  ........./a.....\n00007f808bffe770: 00000000 00000000  00000000 00000011  ................\n00007f808bffe780: 00000000 00000130  00007f80 df60593e  0.......>Y`.....\n00007f808bffe790: 00000000 00000120  00000000 00000000   ...............\n                  ^ SP\n00007f808bffe7a0: 00010000 00000013  00000000 00000000  ................\n00007f808bffe7b0: 00000000 00000000  00007f80 df688cff  ..........h.....\n00007f808bffe7c0: 00007f80 89ffb910  00007f80 df688cb1  ..........h.....\n00007f808bffe7d0: 00007f80 89ffb640  00007f80 8bffeaa0  @...............\n00007f808bffe7e0: 00007f80 8bffe90e  00007f80 8bffe90f  ................\n00007f808bffe7f0: 00007f80 89ffb640  00007f80 df6037f5  @........7`.....\n00007f808bffe800: 00007f80 84000020  00000000 00000120   ....... .......\n00007f808bffe810: 00000000 003d0f00  701e1ebb c8982900  ..=......).....p\n00007f808bffe820: 00007f80 8bfff640  00000000 00000006  @...............\n00007f808bffe830: 00007f80 84000c90  00007f80 8bffeb60  ........`.......\n00007f808bffe840: 00001419 7f1334a0  00007f80 df5b8646  .4......F.[.....\n00007f808bffe850: 00007f80 df775e90  00007f80 df5a27f3  .^w......'Z.....\n00007f808bffe860: 00000000 00000020  701e1ebb c8982900   ........).....p\n00007f808bffe870: 00007f80 897fb000  00007f80 df677ac4  .........zg.....\n00007f808bffe880: 00000000 00000000  00000000 0000000d  ................\nruntime: g 0 gp=0x14197f1334a0: unknown pc 0x7f80df60594c\nstack: frame={sp:0x7f808bffe790, fp:0x0} stack=[0x7f808b7ff1e0,0x7f808bffede0)\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n00007f808bffe690: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6a0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6b0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6c0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6d0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6e0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe6f0: 00000000 00000000  00000000 00000000  ................\n00007f808bffe700: 00000000 00000000  00000000 00000000  ................\n00007f808bffe710: 00000000 00000000  00000000 00000000  ................\n00007f808bffe720: 00000000 00000000  00000000 00000000  ................\n00007f808bffe730: 00000000 00000000  00000000 00000000  ................\n00007f808bffe740: 00000000 00000000  00000000 00000000  ................\n00007f808bffe750: 00000000 00000000  00000000 00000000  ................\n00007f808bffe760: 00000000 00000000  00007f80 df612fa9  ........./a.....\n00007f808bffe770: 00000000 00000000  00000000 00000011  ................\n00007f808bffe780: 00000000 00000130  00007f80 df60593e  0.......>Y`.....\n00007f808bffe790: 00000000 00000120  00000000 00000000   ...............\n                  ^ SP\n00007f808bffe7a0: 00010000 00000013  00000000 00000000  ................\n00007f808bffe7b0: 00000000 00000000  00007f80 df688cff  ..........h.....\n00007f808bffe7c0: 00007f80 89ffb910  00007f80 df688cb1  ..........h.....\n00007f808bffe7d0: 00007f80 89ffb640  00007f80 8bffeaa0  @...............\n00007f808bffe7e0: 00007f80 8bffe90e  00007f80 8bffe90f  ................\n00007f808bffe7f0: 00007f80 89ffb640  00007f80 df6037f5  @........7`.....\n00007f808bffe800: 00007f80 84000020  00000000 00000120   ....... .......\n00007f808bffe810: 00000000 003d0f00  701e1ebb c8982900  ..=......).....p\n00007f808bffe820: 00007f80 8bfff640  00000000 00000006  @...............\n00007f808bffe830: 00007f80 84000c90  00007f80 8bffeb60  ........`.......\n00007f808bffe840: 00001419 7f1334a0  00007f80 df5b8646  .4......F.[.....\n00007f808bffe850: 00007f80 df775e90  00007f80 df5a27f3  .^w......'Z.....\n00007f808bffe860: 00000000 00000020  701e1ebb c8982900   ........).....p\n00007f808bffe870: 00007f80 897fb000  00007f80 df677ac4  .........zg.....\n00007f808bffe880: 00000000 00000000  00000000 0000000d  ................\n\ngoroutine 1 gp=0x14197f1321e0 m=nil [sync.WaitGroup.Wait]:\nruntime.gopark(0x14197fe86c00?, 0x2022a4d?, 0x10?, 0x63?, 0x14197fe86c30?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197ff32be8 sp=0x14197ff32bc8 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x14197fc98610, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197ff32c50 sp=0x14197ff32be8 pc=0x1f162b2\nsync.runtime_SemacquireWaitGroup(0x14197fd289c0?, 0xe0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x14197ff32c88 sp=0x14197ff32c50 pc=0x1f3a66e\nsync.(*WaitGroup).Wait(0x14197fc98608)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x14197ff32cb0 sp=0x14197ff32c88 pc=0x1f4f445\ngolang.org/x/sync/errgroup.(*Group).Wait(0x14197fc98600)\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x14197ff32cd0 sp=0x14197ff32cb0 pc=0x362507e\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x14197f521dc0, {0x59f0b08, 0x14197f530fc0}, {0x14197fcca000, 0x15, 0x1c})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x14197ff32d90 sp=0x14197ff32cd0 pc=0x3927e05\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x14197f521dc0, {0x59f0b08, 0x14197f530fc0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:500 +0x4b08 fp=0x14197ff33968 sp=0x14197ff32d90 pc=0x39261c8\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedQueryOptions).RunQuery(0x14197f1401e0, {0x59f0b08, 0x14197f530fc0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_cmd.go:75 +0x565 fp=0x14197ff33b30 sp=0x14197ff33968 pc=0x3937005\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawQueryOptions).Run(0x0?, {0x59f0b08, 0x14197f530fc0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_options.go:178 +0x54 fp=0x14197ff33b58 sp=0x14197ff33b30 pc=0x3939d34\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryCommand.func1(0x14197f4ba008?, {0x14197f5f82a0?, 0x4?, 0x553b0e2?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_cmd.go:45 +0x28 fp=0x14197ff33b80 sp=0x14197ff33b58 pc=0x3936a68\ngithub.com/spf13/cobra.(*Command).execute(0x14197f4ba008, {0x14197f2c0030, 0xe, 0xf})\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x14197ff33d28 sp=0x14197ff33b80 pc=0x212e614\ngithub.com/spf13/cobra.(*Command).ExecuteC(0x14197f625b08)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x14197ff33e18 sp=0x14197ff33d28 pc=0x212efa5\ngithub.com/spf13/cobra.(*Command).Execute(...)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071\ngithub.com/spf13/cobra.(*Command).ExecuteContext(...)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064\nmain.main()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x14197ff33f48 sp=0x14197ff33e18 pc=0x47ddd6a\nruntime.main()\n\t/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x14197ff33fe0 sp=0x14197ff33f48 pc=0x1f01975\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197ff33fe8 sp=0x14197ff33fe0 pc=0x1f40801\n\ngoroutine 2 gp=0x14197f132d20 m=nil [force gc (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f2fa8 sp=0x14197f1f2f88 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.forcegchelper()\n\t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x14197f1f2fe0 sp=0x14197f1f2fa8 pc=0x1f01c93\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f2fe8 sp=0x14197f1f2fe0 pc=0x1f40801\ncreated by runtime.init.7 in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\n\ngoroutine 18 gp=0x14197f2843c0 m=nil [GC sweep wait]:\nruntime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1ee788 sp=0x14197f1ee768 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.bgsweep(0x14197f292000)\n\t/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x14197f1ee7c8 sp=0x14197f1ee788 pc=0x1ee9f11\nruntime.gcenable.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x14197f1ee7e0 sp=0x14197f1ee7c8 pc=0x1edadd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1ee7e8 sp=0x14197f1ee7e0 pc=0x1f40801\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\n\ngoroutine 19 gp=0x14197f2845a0 m=nil [GC scavenge wait]:\nruntime.gopark(0x138d6d?, 0x3979fd?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1eef78 sp=0x14197f1eef58 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.(*scavengerState).park(0x82979c0)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x14197f1eefa8 sp=0x14197f1eef78 pc=0x1ee77e9\nruntime.bgscavenge(0x14197f292000)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x14197f1eefc8 sp=0x14197f1eefa8 pc=0x1ee7d99\nruntime.gcenable.gowrap2()\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x14197f1eefe0 sp=0x14197f1eefc8 pc=0x1edad97\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1eefe8 sp=0x14197f1eefe0 pc=0x1f40801\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\n\ngoroutine 20 gp=0x14197f284780 m=nil [GOMAXPROCS updater (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1ef788 sp=0x14197f1ef768 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.updateMaxProcsGoroutine()\n\t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x14197f1ef7e0 sp=0x14197f1ef788 pc=0x1f10267\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1ef7e8 sp=0x14197f1ef7e0 pc=0x1f40801\ncreated by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\n\ngoroutine 21 gp=0x14197f284d20 m=nil [finalizer wait]:\nruntime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x14197f1f2601?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f2620 sp=0x14197f1f2600 pc=0x1f38d2e\nruntime.runFinalizers()\n\t/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x14197f1f27e0 sp=0x14197f1f2620 pc=0x1ed9cc7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f27e8 sp=0x14197f1f27e0 pc=0x1f40801\ncreated by runtime.createfing in goroutine 1\n\t/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d\n\ngoroutine 22 gp=0x14197f47c960 m=nil [cleanup wait]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1eff68 sp=0x14197f1eff48 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.(*cleanupQueue).dequeue(0x82980e0)\n\t/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x14197f1effa0 sp=0x14197f1eff68 pc=0x1ed6074\nruntime.runCleanups()\n\t/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x14197f1effe0 sp=0x14197f1effa0 pc=0x1ed66e5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1effe8 sp=0x14197f1effe0 pc=0x1f40801\ncreated by runtime.(*cleanupQueue).createGs in goroutine 1\n\t/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5\n\ngoroutine 23 gp=0x14197f5a30e0 m=nil [GC worker (idle)]:\nruntime.gopark(0x125b862b4ec7?, 0x1?, 0x21?, 0xab?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f0740 sp=0x14197f1f0720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x14197f29d9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f07c8 sp=0x14197f1f0740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f07e0 sp=0x14197f1f07c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f07e8 sp=0x14197f1f07e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 3 gp=0x14197f133680 m=nil [runnable]:\nruntime.gopark(0x125b8625d223?, 0x3?, 0x89?, 0x19?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f3740 sp=0x14197f1f3720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x14197f29d9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f37c8 sp=0x14197f1f3740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f37e0 sp=0x14197f1f37c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f37e8 sp=0x14197f1f37e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 34 gp=0x14197f604000 m=nil [GC worker (idle)]:\nruntime.gopark(0x125b862e8e02?, 0x3?, 0xe6?, 0x2d?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60a740 sp=0x14197f60a720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x14197f29d9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f60a7c8 sp=0x14197f60a740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f60a7e0 sp=0x14197f60a7c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60a7e8 sp=0x14197f60a7e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 24 gp=0x14197f5a32c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x125b862118da?, 0x1?, 0x4d?, 0x9a?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f0f40 sp=0x14197f1f0f20 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x14197f29d9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f0fc8 sp=0x14197f1f0f40 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f0fe0 sp=0x14197f1f0fc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f0fe8 sp=0x14197f1f0fe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 4 gp=0x14197f133860 m=nil [GC worker (idle)]:\nruntime.gopark(0x125b7fe90140?, 0x3?, 0x20?, 0x3a?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f3f40 sp=0x14197f1f3f20 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x14197f29d9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f3fc8 sp=0x14197f1f3f40 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f3fe0 sp=0x14197f1f3fc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f3fe8 sp=0x14197f1f3fe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 35 gp=0x14197f6041e0 m=nil [GC worker (idle)]:\nruntime.gopark(0x125b862a34bd?, 0x3?, 0x4?, 0x15?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60af40 sp=0x14197f60af20 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x14197f29d9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f60afc8 sp=0x14197f60af40 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f60afe0 sp=0x14197f60afc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60afe8 sp=0x14197f60afe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 25 gp=0x14197f5a34a0 m=3 mp=0x14197f1a5808 [GC mark termination]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x14197f1f16a0 sp=0x14197f1f1690 pc=0x1f3ebc8\nruntime.forEachP(...)\n\t/usr/lib/golang/src/runtime/proc.go:2112\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x14197f1f1740 sp=0x14197f1f16a0 pc=0x1edbc34\nruntime.gcBgMarkWorker(0x14197f29d9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x14197f1f17c8 sp=0x14197f1f1740 pc=0x1edda88\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f17e0 sp=0x14197f1f17c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f17e8 sp=0x14197f1f17e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 5 gp=0x14197f133a40 m=nil [GC worker (idle)]:\nruntime.gopark(0x125b8624ea9b?, 0x1?, 0xfd?, 0x7a?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f4740 sp=0x14197f1f4720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x14197f29d9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f47c8 sp=0x14197f1f4740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f47e0 sp=0x14197f1f47c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f47e8 sp=0x14197f1f47e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 50 gp=0x14197f768f00 m=nil [select, locked to thread]:\nruntime.gopark(0x14197f60c7a8?, 0x2?, 0x1?, 0x8f?, 0x14197f60c794?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60c628 sp=0x14197f60c608 pc=0x1f38d2e\nruntime.selectgo(0x14197f60c7a8, 0x14197f60c790, 0x0?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f60c758 sp=0x14197f60c628 pc=0x1f152a5\nruntime.ensureSigM.func1()\n\t/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x14197f60c7e0 sp=0x14197f60c758 pc=0x1f328e8\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60c7e8 sp=0x14197f60c7e0 pc=0x1f40801\ncreated by runtime.ensureSigM in goroutine 1\n\t/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5\n\ngoroutine 26 gp=0x14197f3b34a0 m=7 mp=0x14197f55b008 [syscall]:\nruntime.notetsleepg(0x82bdea0, 0xffffffffffffffff)\n\t/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x14197f6077a0 sp=0x14197f607778 pc=0x1ece509\nos/signal.signal_recv()\n\t/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x14197f6077c0 sp=0x14197f6077a0 pc=0x1f3b298\nos/signal.loop()\n\t/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x14197f6077e0 sp=0x14197f6077c0 pc=0x2030553\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f6077e8 sp=0x14197f6077e0 pc=0x1f40801\ncreated by os/signal.Notify.func1.1 in goroutine 1\n\t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\n\ngoroutine 27 gp=0x14197f6894a0 m=nil [select]:\nruntime.gopark(0x14197f607f98?, 0x2?, 0x68?, 0x81?, 0x14197f607f8c?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f607e28 sp=0x14197f607e08 pc=0x1f38d2e\nruntime.selectgo(0x14197f607f98, 0x14197f607f88, 0x0?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f607f58 sp=0x14197f607e28 pc=0x1f152a5\nos/signal.NotifyContext.func1()\n\t/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x14197f607fe0 sp=0x14197f607f58 pc=0x202ff9c\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f607fe8 sp=0x14197f607fe0 pc=0x1f40801\ncreated by os/signal.NotifyContext in goroutine 1\n\t/usr/lib/golang/src/os/signal/signal.go:291 +0x158\n\ngoroutine 224 gp=0x14197f689680 m=nil [semacquire]:\nruntime.gopark(0x7f80904adba8?, 0x1eed7e0?, 0xc0?, 0x68?, 0x4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419800b5b58 sp=0x1419800b5b38 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x1419800b5bc0 sp=0x1419800b5b58 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x1419800b5c60 sp=0x1419800b5bc0 pc=0x1edbb12\nruntime.gcAssistAlloc(0x14197f689680)\n\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x1419800b5cf0 sp=0x1419800b5c60 pc=0x1ee06b6\nruntime.deductAssistCredit(0x7ffce51f9080?)\n\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x1419800b5d10 sp=0x1419800b5cf0 pc=0x1ed10a6\nruntime.mallocgc(0x7fe00, 0x4af32a0, 0x1)\n\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x1419800b5d40 sp=0x1419800b5d10 pc=0x1f35e07\nruntime.makeslice(0x14197fc9d0b0?, 0x141983a1fe00?, 0x20000?)\n\t/usr/lib/golang/src/runtime/slice.go:117 +0x49 fp=0x1419800b5d68 sp=0x1419800b5d40 pc=0x1f3b489\nencoding/json.(*Decoder).refill(0x14197f658c80)\n\t/usr/lib/golang/src/encoding/json/stream.go:161 +0xee fp=0x1419800b5db8 sp=0x1419800b5d68 pc=0x200d48e\nencoding/json.(*Decoder).readValue(0x14197f658c80)\n\t/usr/lib/golang/src/encoding/json/stream.go:142 +0x85 fp=0x1419800b5e08 sp=0x1419800b5db8 pc=0x200d125\nencoding/json.(*Decoder).Decode(0x14197f658c80, {0x4cfb340, 0x14197ffa44c0})\n\t/usr/lib/golang/src/encoding/json/stream.go:65 +0x75 fp=0x1419800b5e38 sp=0x1419800b5e08 pc=0x200cdf5\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readPrimaryTable(0x14197f8c54f0, 0x14197fb50000)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:264 +0x16e fp=0x1419800b5ec8 sp=0x1419800b5e38 pc=0x38d822e\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x14197f8c54f0)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:174 +0x189 fp=0x1419800b5f30 sp=0x1419800b5ec8 pc=0x38d7b89\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x14197f8c54f0, 0x14198007be90)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x1419800b5fc0 sp=0x1419800b5f30 pc=0x38d788c\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x1419800b5fe0 sp=0x1419800b5fc0 pc=0x38d751b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1419800b5fe8 sp=0x1419800b5fe0 pc=0x1f40801\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 183\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc\n\ngoroutine 87 gp=0x14197fd3c000 m=nil [sync.WaitGroup.Wait]:\nruntime.gopark(0x14197fcf9e00?, 0x1ed0734?, 0x20?, 0x94?, 0x7f80917bdf06?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf9da0 sp=0x14197fcf9d80 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x14197f69cc50, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fcf9e08 sp=0x14197fcf9da0 pc=0x1f162b2\nsync.runtime_SemacquireWaitGroup(0x14197f86b2d8?, 0x0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x14197fcf9e40 sp=0x14197fcf9e08 pc=0x1f3a66e\nsync.(*WaitGroup).Wait(0x14197f69cc48)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x14197fcf9e68 sp=0x14197fcf9e40 pc=0x1f4f445\ngolang.org/x/sync/errgroup.(*Group).Wait(0x14197f69cc40)\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x14197fcf9e88 sp=0x14197fcf9e68 pc=0x362507e\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x14197f521d80, {0x59f0b40, 0x14197f8c4b40}, {0x14197fcca000, 0x15, 0x6f63746669687372?}, 0x14197f1c02a0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x14197fcf9f00 sp=0x14197fcf9e88 pc=0x392976a\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x14197fcf9f78 sp=0x14197fcf9f00 pc=0x392808e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197fcf9fe0 sp=0x14197fcf9f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf9fe8 sp=0x14197fcf9fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 77 gp=0x14197fd3c1e0 m=nil [IO wait]:\nruntime.gopark(0x14197fb3e4a0?, 0x1edb176?, 0x80?, 0xed?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f202808 sp=0x14197f2027e8 pc=0x1f38d2e\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x14197f202840 sp=0x14197f202808 pc=0x1ef9e57\ninternal/poll.runtime_pollWait(0x7f8098109000, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x14197f202860 sp=0x14197f202840 pc=0x1f375c5\ninternal/poll.(*pollDesc).wait(0x14197f323a80?, 0x14197f34d300?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x14197f202888 sp=0x14197f202860 pc=0x1fbd167\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0x14197f323a80, {0x14197f34d300, 0x1300, 0x1300})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x14197f202920 sp=0x14197f202888 pc=0x1fbe4ce\nnet.(*netFD).Read(0x14197f323a80, {0x14197f34d300?, 0x14197f34d30a?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x14197f202968 sp=0x14197f202920 pc=0x20798c5\nnet.(*conn).Read(0x14197f1300d8, {0x14197f34d300?, 0x7f809057a868?, 0x7f80df53a6d8?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x14197f2029b0 sp=0x14197f202968 pc=0x208b7a5\ncrypto/tls.(*atLeastReader).Read(0x14197f821a10, {0x14197f34d300?, 0x14197f202a98?, 0x1efcc28?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x14197f2029f8 sp=0x14197f2029b0 pc=0x2222ddb\nbytes.(*Buffer).ReadFrom(0x14197f8a57a8, {0x59a3680, 0x14197f821a10})\n\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x14197f202a50 sp=0x14197f2029f8 pc=0x1fe9258\ncrypto/tls.(*Conn).readFromUntil(0x14197f8a5508, {0x59a32c0, 0x14197f1300d8}, 0x222764c?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x14197f202a88 sp=0x14197f202a50 pc=0x2222fbe\ncrypto/tls.(*Conn).readRecordOrCCS(0x14197f8a5508, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x14197f202c68 sp=0x14197f202a88 pc=0x222009b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:588\ncrypto/tls.(*Conn).Read(0x14197f8a5508, {0x14197fa26000, 0x1000, 0x4e55540?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x14197f202cd0 sp=0x14197f202c68 pc=0x22267c5\nnet/http.(*persistConn).Read(0x14197f8032c0, {0x14197fa26000?, 0x59a0ea0?, 0x81260e0?})\n\t/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x14197f202d30 sp=0x14197f202cd0 pc=0x2319d07\nbufio.(*Reader).fill(0x14197f29b620)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f202d68 sp=0x14197f202d30 pc=0x20a6c63\nbufio.(*Reader).Peek(0x14197f29b620, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x14197f202d80 sp=0x14197f202d68 pc=0x20a6df3\nnet/http.(*persistConn).readLoop(0x14197f8032c0)\n\t/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x14197f202fc8 sp=0x14197f202d80 pc=0x231a812\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x14197f202fe0 sp=0x14197f202fc8 pc=0x2319197\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f202fe8 sp=0x14197f202fe0 pc=0x1f40801\ncreated by net/http.(*Transport).dialConn in goroutine 9\n\t/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac\n\ngoroutine 7 gp=0x14197fe025a0 m=nil [sync.Mutex.Lock]:\nruntime.gopark(0x14197f9d3c68?, 0x14197f9d3c70?, 0x94?, 0x69?, 0x14197f9d3c78?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f9d3c00 sp=0x14197f9d3be0 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x14197fccaad4, 0x0, 0x3, 0x2, 0x16)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197f9d3c68 sp=0x14197f9d3c00 pc=0x1f162b2\ninternal/sync.runtime_SemacquireMutex(0x1000?, 0xd0?, 0x1ed11a5?)\n\t/usr/lib/golang/src/runtime/sema.go:95 +0x25 fp=0x14197f9d3ca0 sp=0x14197f9d3c68 pc=0x1f3a545\ninternal/sync.(*Mutex).lockSlow(0x14197fccaad0)\n\t/usr/lib/golang/src/internal/sync/mutex.go:149 +0x15d fp=0x14197f9d3cf0 sp=0x14197f9d3ca0 pc=0x1f4d31d\ninternal/sync.(*Mutex).Lock(...)\n\t/usr/lib/golang/src/internal/sync/mutex.go:70\nsync.(*Mutex).Lock(...)\n\t/usr/lib/golang/src/sync/mutex.go:46\nnet/http.(*http2clientConnReadLoop).streamByID(0x22babb5?, 0x1b, 0x1)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:10083 +0x5b fp=0x14197f9d3d48 sp=0x14197f9d3cf0 pc=0x22e10db\nnet/http.(*http2clientConnReadLoop).processData(0x14197f9d3fa8, 0x14197fc9d200)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9939 +0x45 fp=0x14197f9d3e18 sp=0x14197f9d3d48 pc=0x22e05e5\nnet/http.(*http2clientConnReadLoop).run(0x14197f9d3fa8)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9584 +0x4b6 fp=0x14197f9d3f60 sp=0x14197f9d3e18 pc=0x22de536\nnet/http.(*http2ClientConn).readLoop(0x14197fccaa80)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x14197f9d3fc8 sp=0x14197f9d3f60 pc=0x22dd532\nnet/http.(*http2Transport).newClientConn.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x14197f9d3fe0 sp=0x14197f9d3fc8 pc=0x22d7057\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f9d3fe8 sp=0x14197f9d3fe0 pc=0x1f40801\ncreated by net/http.(*http2Transport).newClientConn in goroutine 6\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5\n\ngoroutine 78 gp=0x14197fe02960 m=nil [select]:\nruntime.gopark(0x14197f9f7f48?, 0x2?, 0x0?, 0xb6?, 0x14197f9f7ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf2d88 sp=0x14197fcf2d68 pc=0x1f38d2e\nruntime.selectgo(0x14197fcf2f48, 0x14197f9f7ef0, 0x14197f1cd0c0?, 0x0, 0x14197f89c2a0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197fcf2eb8 sp=0x14197fcf2d88 pc=0x1f152a5\nnet/http.(*persistConn).writeLoop(0x14197f8032c0)\n\t/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x14197fcf2fc8 sp=0x14197fcf2eb8 pc=0x231c426\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x14197fcf2fe0 sp=0x14197fcf2fc8 pc=0x2319157\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf2fe8 sp=0x14197fcf2fe0 pc=0x1f40801\ncreated by net/http.(*Transport).dialConn in goroutine 9\n\t/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe\n\ngoroutine 163 gp=0x14197f7694a0 m=nil [select]:\nruntime.gopark(0x14197fdd3f08?, 0x6?, 0xa0?, 0x94?, 0x14197fdd3da4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f205c08 sp=0x14197f205be8 pc=0x1f38d2e\nruntime.selectgo(0x14197f205f08, 0x14197fdd3d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f205d38 sp=0x14197f205c08 pc=0x1f152a5\nnet/http.(*http2clientStream).writeRequest(0x14197f37f380, 0x14197f49e140, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x14197f205f90 sp=0x14197f205d38 pc=0x22da6a8\nnet/http.(*http2clientStream).doRequest(0x14197f37f380, 0x14197f546100?, 0x14197f609798?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x14197f205fb8 sp=0x14197f205f90 pc=0x22d99d8\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197f205fe0 sp=0x14197f205fb8 pc=0x22d999f\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f205fe8 sp=0x14197f205fe0 pc=0x1f40801\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 184\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n\ngoroutine 88 gp=0x14197f769a40 m=nil [semacquire]:\nruntime.gopark(0x141980119958?, 0x1ed3e05?, 0xe8?, 0xd2?, 0x7f80904ad2e8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x141980119908 sp=0x1419801198e8 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x141980119970 sp=0x141980119908 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x141980119a10 sp=0x141980119970 pc=0x1edbb12\nruntime.gcAssistAlloc(0x14197f769a40)\n\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x141980119aa0 sp=0x141980119a10 pc=0x1ee06b6\nruntime.deductAssistCredit(0x141983b00000?)\n\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x141980119ac0 sp=0x141980119aa0 pc=0x1ed10a6\nruntime.mallocgc(0x7edd, 0x0, 0x0)\n\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x141980119af0 sp=0x141980119ac0 pc=0x1f35e07\nruntime.slicebytetostring(0x4e537a0?, 0x141983b00000, 0x7edd)\n\t/usr/lib/golang/src/runtime/string.go:170 +0x77 fp=0x141980119b20 sp=0x141980119af0 pc=0x1f1e397\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x14197f8c4b40}, 0x14197f1c03f0, 0x14197f531740)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:322 +0x91b fp=0x141980119f48 sp=0x141980119b20 pc=0x3920bfb\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x141980119f78 sp=0x141980119f48 pc=0x392800c\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x141980119fe0 sp=0x141980119f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x141980119fe8 sp=0x141980119fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 110 gp=0x14197fd3cb40 m=nil [sync.Mutex.Lock]:\nruntime.gopark(0x22b8a73?, 0x14197f7e1000?, 0xb0?, 0x4c?, 0x8160a40?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1c7510 sp=0x14197f1c74f0 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x14197fccaad4, 0x0, 0x3, 0x2, 0x16)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197f1c7578 sp=0x14197f1c7510 pc=0x1f162b2\ninternal/sync.runtime_SemacquireMutex(0x14197f1c75d0?, 0x66?, 0x61f?)\n\t/usr/lib/golang/src/runtime/sema.go:95 +0x25 fp=0x14197f1c75b0 sp=0x14197f1c7578 pc=0x1f3a545\ninternal/sync.(*Mutex).lockSlow(0x14197fccaad0)\n\t/usr/lib/golang/src/internal/sync/mutex.go:149 +0x15d fp=0x14197f1c7600 sp=0x14197f1c75b0 pc=0x1f4d31d\ninternal/sync.(*Mutex).Lock(...)\n\t/usr/lib/golang/src/internal/sync/mutex.go:70\nsync.(*Mutex).Lock(...)\n\t/usr/lib/golang/src/sync/mutex.go:46\nnet/http.http2transportResponseBody.Read({0x1f3ec12?}, {0x14198313d000?, 0x1eed797?, 0x14197f1c76b0?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9875 +0x209 fp=0x14197f1c7688 sp=0x14197f1c7600 pc=0x22dff89\nbufio.(*Reader).fill(0x14197f3a00c0)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f1c76c0 sp=0x14197f1c7688 pc=0x20a6c63\nbufio.(*Reader).ReadByte(0x14197f3a00c0)\n\t/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x14197f1c76d8 sp=0x14197f1c76c0 pc=0x20a73c7\ncompress/flate.(*decompressor).moreBits(0x141980059908)\n\t/usr/lib/golang/src/compress/flate/inflate.go:697 +0x24 fp=0x14197f1c7710 sp=0x14197f1c76d8 pc=0x2288824\ncompress/flate.(*decompressor).nextBlock(0x141980059908)\n\t/usr/lib/golang/src/compress/flate/inflate.go:304 +0x28 fp=0x14197f1c7728 sp=0x14197f1c7710 pc=0x2286ba8\ncompress/flate.(*decompressor).Read(0x141980059908, {0x14198314e000, 0x1000, 0x7f8090ef0f00?})\n\t/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x14197f1c7758 sp=0x14197f1c7728 pc=0x2286dbb\ncompress/gzip.(*Reader).Read(0x14197f672dc8, {0x14198314e000, 0x1000, 0x1000})\n\t/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x14197f1c77e8 sp=0x14197f1c7758 pc=0x228c838\ngithub.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x410d0fa76c5d7d40?, {0x14198314e000?, 0x7f8090ef0d20?, 0x52ffb40?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x14197f1c7818 sp=0x14197f1c77e8 pc=0x38b33e2\nbufio.(*Reader).fill(0x14197f3a0120)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f1c7850 sp=0x14197f1c7818 pc=0x20a6c63\nbufio.(*Reader).Peek(0x14197f3a0120, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x14197f1c7868 sp=0x14197f1c7850 pc=0x20a6df3\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.validateJsonResponse(0x14197f3a0120)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:38 +0x25 fp=0x14197f1c78b8 sp=0x14197f1c7868 pc=0x38d6be5\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.newFrameReader({0x59db110, 0x14197fe1e040}, {0x59f0b40, 0x14197f748140})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:26 +0x188 fp=0x14197f1c7960 sp=0x14197f1c78b8 pc=0x38d6a08\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset({0x59f0bb0?, 0x14197f4797a0?}, {0x59db110, 0x14197fe1e040}, 0x1, 0x3e8, 0x1)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:66 +0x19e fp=0x14197f1c79e0 sp=0x14197f1c7960 pc=0x38d737e\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x14197f521d40, {0x59f0bb0, 0x14197f4797a0}, {0x14197f49a330, 0xb}, 0x14197fe1e700, {0x0?, 0x2?, 0x0?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:167 +0x2b7 fp=0x14197f1c7a78 sp=0x14197f1c79e0 pc=0x38e4f17\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197f7a2600}, 0x14197f1c02a0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x14197f1c7d60 sp=0x14197f1c7a78 pc=0x391b5c5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x14197f521d80, {0x59f0b40, 0x14197fdd8140}, {0x5a00340, 0x14197fc1e5d0}, 0x14197f1c02a0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:98 +0x366 fp=0x14197f1c7e88 sp=0x14197f1c7d60 pc=0x3929ec6\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14197f1c7f78 sp=0x14197f1c7e88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197f1c7fe0 sp=0x14197f1c7f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1c7fe8 sp=0x14197f1c7fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 89 gp=0x14197fd82960 m=nil [select]:\nruntime.gopark(0x1419800b7d00?, 0x2?, 0xd7?, 0xfd?, 0x1419800b7c74?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419800b7ac8 sp=0x1419800b7aa8 pc=0x1f38d2e\nruntime.selectgo(0x1419800b7d00, 0x1419800b7c70, 0x1419801521c0?, 0x0, 0x14197f5daba0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x1419800b7bf8 sp=0x1419800b7ac8 pc=0x1f152a5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x14197f521d80?, {0x59f0b40, 0x14197f8c4b40}, 0x14197f1c02a0, 0x14197f1c03f0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x1419800b7f10 sp=0x1419800b7bf8 pc=0x392836e\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x1419800b7f78 sp=0x1419800b7f10 pc=0x3927f3d\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x1419800b7fe0 sp=0x1419800b7f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1419800b7fe8 sp=0x1419800b7fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 175 gp=0x14197fd3d2c0 m=nil [sync.Cond.Wait]:\nruntime.gopark(0x22c2d5b?, 0x14197fe1c080?, 0x0?, 0xe0?, 0x14197f777000?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14198012b978 sp=0x14198012b958 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nsync.runtime_notifyListWait(0x14197f37f3c8, 0x3c)\n\t/usr/lib/golang/src/runtime/sema.go:617 +0x1b3 fp=0x14198012b9c8 sp=0x14198012b978 pc=0x1f3a8f3\nsync.(*Cond).Wait(0x1000?)\n\t/usr/lib/golang/src/sync/cond.go:71 +0x73 fp=0x14198012ba00 sp=0x14198012b9c8 pc=0x1f4d673\nnet/http.(*http2pipe).Read(0x14197f37f3b0, {0x14197ff7e000, 0x1000, 0x1000})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:3928 +0xdb fp=0x14198012ba68 sp=0x14198012ba00 pc=0x22c2c1b\nnet/http.http2transportResponseBody.Read({0x4a54?}, {0x14197ff7e000?, 0x14198012bb80?, 0x2287949?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9852 +0x59 fp=0x14198012baf0 sp=0x14198012ba68 pc=0x22dfdd9\nbufio.(*Reader).fill(0x14197fcaa000)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14198012bb28 sp=0x14198012baf0 pc=0x20a6c63\nbufio.(*Reader).ReadByte(0x14197fcaa000)\n\t/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x14198012bb40 sp=0x14198012bb28 pc=0x20a73c7\ncompress/flate.(*decompressor).moreBits(0x141980056008)\n\t/usr/lib/golang/src/compress/flate/inflate.go:697 +0x24 fp=0x14198012bb78 sp=0x14198012bb40 pc=0x2288824\ncompress/flate.(*decompressor).nextBlock(0x141980056008)\n\t/usr/lib/golang/src/compress/flate/inflate.go:304 +0x28 fp=0x14198012bb90 sp=0x14198012bb78 pc=0x2286ba8\ncompress/flate.(*decompressor).Read(0x141980056008, {0x141980034b13, 0x4ed, 0xb13?})\n\t/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x14198012bbc0 sp=0x14198012bb90 pc=0x2286dbb\ncompress/gzip.(*Reader).Read(0x141980000b08, {0x141980034b13, 0x4ed, 0x4ed})\n\t/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x14198012bc50 sp=0x14198012bbc0 pc=0x228c838\ngithub.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x14198012bcf0?, {0x141980034b13?, 0x1000?, 0x0?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x14198012bc80 sp=0x14198012bc50 pc=0x38b33e2\nbufio.(*Reader).fill(0x141980807010)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14198012bcb8 sp=0x14198012bc80 pc=0x20a6c63\nbufio.(*Reader).ReadSlice(0x141980807010, 0xa)\n\t/usr/lib/golang/src/bufio/bufio.go:377 +0x29 fp=0x14198012bd00 sp=0x14198012bcb8 pc=0x20a7769\nbufio.(*Reader).collectFragments(0x141980807010, 0xa)\n\t/usr/lib/golang/src/bufio/bufio.go:452 +0x85 fp=0x14198012bde0 sp=0x14198012bd00 pc=0x20a7b85\nbufio.(*Reader).ReadBytes(0x4bbc5a0?, 0xa0?)\n\t/usr/lib/golang/src/bufio/bufio.go:479 +0x17 fp=0x14198012be60 sp=0x14198012bde0 pc=0x20a7e77\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.(*frameReader).advance(0x141980807000)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:64 +0x45 fp=0x14198012beb0 sp=0x14198012be60 pc=0x38d6d45\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readRoutine(0x141980807000, 0x14197f8c8370)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:85 +0x51 fp=0x14198012bfc0 sp=0x14198012beb0 pc=0x38d7591\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap2()\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x1b fp=0x14198012bfe0 sp=0x14198012bfc0 pc=0x38d74db\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14198012bfe8 sp=0x14198012bfe0 pc=0x1f40801\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 184\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x24a\n\ngoroutine 183 gp=0x14197fd3d860 m=nil [semacquire]:\nruntime.gopark(0x7f8090f36c08?, 0x1eed7e0?, 0x80?, 0x43?, 0x2?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fac5208 sp=0x14197fac51e8 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fac5270 sp=0x14197fac5208 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x14197fac5310 sp=0x14197fac5270 pc=0x1edbb12\nruntime.gcAssistAlloc(0x14197fd3d860)\n\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x14197fac53a0 sp=0x14197fac5310 pc=0x1ee06b6\nruntime.deductAssistCredit(0x14197fac53f0?)\n\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x14197fac53c0 sp=0x14197fac53a0 pc=0x1ed10a6\nruntime.mallocgc(0x5500, 0x0, 0x0)\n\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x14197fac53f0 sp=0x14197fac53c0 pc=0x1f35e07\nruntime.growslice(0x141983adc000, 0x14197fac54a8?, 0x1ed3be5?, 0x82bed80?, 0x1ed0417?)\n\t/usr/lib/golang/src/runtime/slice.go:265 +0x573 fp=0x14197fac5460 sp=0x14197fac53f0 pc=0x1f3ba93\nstrings.(*Builder).Write(0x14197f569ea0, {0x1419839fd000, 0x1000, 0x1000000010765?})\n\t/usr/lib/golang/src/strings/builder.go:89 +0x90 fp=0x14197fac54b0 sp=0x14197fac5460 pc=0x1ff0cb0\nbufio.(*Writer).Flush(0x14197fac5698)\n\t/usr/lib/golang/src/bufio/bufio.go:642 +0x52 fp=0x14197fac54f8 sp=0x14197fac54b0 pc=0x20a8832\nbufio.(*Writer).WriteString(0x14197fac5698, {0x553a4f6?, 0x553a9e6?})\n\t/usr/lib/golang/src/bufio/bufio.go:766 +0x165 fp=0x14197fac5568 sp=0x14197fac54f8 pc=0x20a8f65\nencoding/csv.(*Writer).Write(0x14197fac5688, {0x14198384d480?, 0x4, 0x141983909500?})\n\t/usr/lib/golang/src/encoding/csv/writer.go:92 +0x2b4 fp=0x14197fac55d0 sp=0x14197fac5568 pc=0x20ad3d4\ngithub.com/Azure/azure-kusto-go/azkustodata/query.(*row).String(0x5e?)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/row_impl.go:101 +0x21f fp=0x14197fac56e8 sp=0x14197fac55d0 pc=0x38cf67f\nfmt.(*pp).handleMethods(0x14197fc20000, 0x7fac58f0?)\n\t/usr/lib/golang/src/fmt/print.go:674 +0x2f4 fp=0x14197fac58a8 sp=0x14197fac56e8 pc=0x1fdd1d4\nfmt.(*pp).printArg(0x14197fc20000, {0x549a3a0, 0x14197fd92e80}, 0x76)\n\t/usr/lib/golang/src/fmt/print.go:757 +0x42e fp=0x14197fac5938 sp=0x14197fac58a8 pc=0x1fdd9ee\nfmt.(*pp).doPrintf(0x14197fc20000, {0x553a400, 0x2}, {0x14197fac5d08, 0x1, 0x1})\n\t/usr/lib/golang/src/fmt/print.go:1075 +0x3ec fp=0x14197fac5a20 sp=0x14197fac5938 pc=0x1fe04ec\nfmt.Sprintf({0x553a400, 0x2}, {0x14197fac5d08, 0x1, 0x1})\n\t/usr/lib/golang/src/fmt/print.go:239 +0x53 fp=0x14197fac5a78 sp=0x14197fac5a20 pc=0x1fda6b3\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197fc4b200}, 0x14197f1c02a0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:160 +0x8d8 fp=0x14197fac5d60 sp=0x14197fac5a78 pc=0x391bad8\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x14197fdd8140?}, {0x5a00300?, 0x14197fc4b200?}, 0x0?)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x14197fac5e88 sp=0x14197fac5d60 pc=0x3929c16\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14197fac5f78 sp=0x14197fac5e88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197fac5fe0 sp=0x14197fac5f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fac5fe8 sp=0x14197fac5fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 184 gp=0x14197fd3da40 m=nil [chan receive]:\nruntime.gopark(0x14197f3c29a0?, 0x2034990?, 0x10?, 0xea?, 0x59f0ad0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14198011f9d8 sp=0x14198011f9b8 pc=0x1f38d2e\nruntime.chanrecv(0x14197f3fa700, 0x14198011fcb8, 0x1)\n\t/usr/lib/golang/src/runtime/chan.go:667 +0x4ae fp=0x14198011fa50 sp=0x14198011f9d8 pc=0x1ec7d2e\nruntime.chanrecv2(0x5a09ed0?, 0x14197f530f90?)\n\t/usr/lib/golang/src/runtime/chan.go:514 +0x12 fp=0x14198011fa78 sp=0x14198011fa50 pc=0x1ec7872\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197fc4b340}, 0x14197f1c02a0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:141 +0x94d fp=0x14198011fd60 sp=0x14198011fa78 pc=0x391bb4d\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x74666968736e6570?, {0x59f0b40?, 0x14197fdd8140?}, {0x5a00300?, 0x14197fc4b340?}, 0x6d736f63222c2273?)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x14198011fe88 sp=0x14198011fd60 pc=0x3929c16\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14198011ff78 sp=0x14198011fe88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14198011ffe0 sp=0x14198011ff78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14198011ffe8 sp=0x14198011ffe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 135 gp=0x14197fb465a0 m=nil [semacquire]:\nruntime.gopark(0x14197f454000?, 0x22dc6c0?, 0xa0?, 0x82?, 0x1419808322a0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fe99ce8 sp=0x14197fe99cc8 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fe99d50 sp=0x14197fe99ce8 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x14197fe99df0 sp=0x14197fe99d50 pc=0x1edbb12\nruntime.gcAssistAlloc(0x14197fb465a0)\n\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x14197fe99e80 sp=0x14197fe99df0 pc=0x1ee06b6\nruntime.deductAssistCredit(0x0?)\n\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x14197fe99ea0 sp=0x14197fe99e80 pc=0x1ed10a6\nruntime.mallocgc(0x70, 0x0, 0x1)\n\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x14197fe99ed0 sp=0x14197fe99ea0 pc=0x1f35e07\nruntime.makechan(0x0?, 0x0)\n\t/usr/lib/golang/src/runtime/chan.go:99 +0x126 fp=0x14197fe99f10 sp=0x14197fe99ed0 pc=0x1ec6826\nnet/http.(*http2clientStream).cleanupWriteRequest(0x14197f2f1200, {0x0, 0x0})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8951 +0xd6 fp=0x14197fe99f90 sp=0x14197fe99f10 pc=0x22db1b6\nnet/http.(*http2clientStream).doRequest(0x14197f2f1200, 0x14197f7480a0?, 0x14197f4d0020?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8719 +0x28 fp=0x14197fe99fb8 sp=0x14197fe99f90 pc=0x22d99e8\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197fe99fe0 sp=0x14197fe99fb8 pc=0x22d999f\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fe99fe8 sp=0x14197fe99fe0 pc=0x1f40801\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 110\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n\ngoroutine 90 gp=0x14197fd82b40 m=nil [select]:\nruntime.gopark(0x14197ff1df08?, 0x6?, 0x40?, 0x2b?, 0x14197ff1dda4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f203c08 sp=0x14197f203be8 pc=0x1f38d2e\nruntime.selectgo(0x14197f203f08, 0x14197ff1dd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f203d38 sp=0x14197f203c08 pc=0x1f152a5\nnet/http.(*http2clientStream).writeRequest(0x14197f2f0f00, 0x14197fb50140, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x14197f203f90 sp=0x14197f203d38 pc=0x22da6a8\nnet/http.(*http2clientStream).doRequest(0x14197f2f0f00, 0x0?, 0x14197f1f1f98?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x14197f203fb8 sp=0x14197f203f90 pc=0x22d99d8\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197f203fe0 sp=0x14197f203fb8 pc=0x22d999f\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f203fe8 sp=0x14197f203fe0 pc=0x1f40801\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 183\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n\ngoroutine 225 gp=0x14197fd834a0 m=nil [sync.Cond.Wait]:\nruntime.gopark(0x22b8a73?, 0x141980390000?, 0x0?, 0x60?, 0xe?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419801298f8 sp=0x1419801298d8 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nsync.runtime_notifyListWait(0x14197f2f0f48, 0x3c)\n\t/usr/lib/golang/src/runtime/sema.go:617 +0x1b3 fp=0x141980129948 sp=0x1419801298f8 pc=0x1f3a8f3\nsync.(*Cond).Wait(0x14197fbb1640?)\n\t/usr/lib/golang/src/sync/cond.go:71 +0x73 fp=0x141980129980 sp=0x141980129948 pc=0x1f4d673\nnet/http.(*http2pipe).Read(0x14197f2f0f30, {0x14197fd98000, 0x1000, 0x1000})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:3928 +0xdb fp=0x1419801299e8 sp=0x141980129980 pc=0x22c2c1b\nnet/http.http2transportResponseBody.Read({0x141980129a60?}, {0x14197fd98000?, 0x20a6c63?, 0x1f3ec12?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9852 +0x59 fp=0x141980129a70 sp=0x1419801299e8 pc=0x22dfdd9\nbufio.(*Reader).fill(0x14197fc90180)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x141980129aa8 sp=0x141980129a70 pc=0x20a6c63\nbufio.(*Reader).ReadByte(0x14197fc90180)\n\t/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x141980129ac0 sp=0x141980129aa8 pc=0x20a73c7\ncompress/flate.(*decompressor).huffSym(0x141980058608, 0x141980058638)\n\t/usr/lib/golang/src/compress/flate/inflate.go:720 +0x7e fp=0x141980129b10 sp=0x141980129ac0 pc=0x228893e\ncompress/flate.(*decompressor).huffmanBlock(0x82bed80?)\n\t/usr/lib/golang/src/compress/flate/inflate.go:495 +0x45 fp=0x141980129b90 sp=0x141980129b10 pc=0x2287605\ncompress/flate.(*decompressor).Read(0x141980058608, {0x14197fd99cea, 0x316, 0xcea?})\n\t/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x141980129bc0 sp=0x141980129b90 pc=0x2286dbb\ncompress/gzip.(*Reader).Read(0x14197f3d8848, {0x14197fd99cea, 0x316, 0x316})\n\t/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x141980129c50 sp=0x141980129bc0 pc=0x228c838\ngithub.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x141980129cf0?, {0x14197fd99cea?, 0x1000?, 0x0?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x141980129c80 sp=0x141980129c50 pc=0x38b33e2\nbufio.(*Reader).fill(0x14197f546110)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x141980129cb8 sp=0x141980129c80 pc=0x20a6c63\nbufio.(*Reader).ReadSlice(0x14197f546110, 0xa)\n\t/usr/lib/golang/src/bufio/bufio.go:377 +0x29 fp=0x141980129d00 sp=0x141980129cb8 pc=0x20a7769\nbufio.(*Reader).collectFragments(0x14197f546110, 0xa)\n\t/usr/lib/golang/src/bufio/bufio.go:452 +0x85 fp=0x141980129de0 sp=0x141980129d00 pc=0x20a7b85\nbufio.(*Reader).ReadBytes(0x4bbc5a0?, 0xa0?)\n\t/usr/lib/golang/src/bufio/bufio.go:479 +0x17 fp=0x141980129e60 sp=0x141980129de0 pc=0x20a7e77\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.(*frameReader).advance(0x14197f546100)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:64 +0x45 fp=0x141980129eb0 sp=0x141980129e60 pc=0x38d6d45\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readRoutine(0x14197f546100, 0x14197f8c54f0)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:85 +0x51 fp=0x141980129fc0 sp=0x141980129eb0 pc=0x38d7591\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap2()\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x1b fp=0x141980129fe0 sp=0x141980129fc0 pc=0x38d74db\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x141980129fe8 sp=0x141980129fe0 pc=0x1f40801\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 183\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x24a\n\ngoroutine 174 gp=0x14197ff64000 m=nil [select]:\nruntime.gopark(0x1419800b9df8?, 0x2?, 0x0?, 0x0?, 0x1419800b9db4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf3c50 sp=0x14197fcf3c30 pc=0x1f38d2e\nruntime.selectgo(0x14197fcf3df8, 0x1419800b9db0, 0x59d87c8?, 0x0, 0x14197f2980e0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197fcf3d80 sp=0x14197fcf3c50 pc=0x1f152a5\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.nextFrame(0x14197f8c8370)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:198 +0x96 fp=0x14197fcf3e38 sp=0x14197fcf3d80 pc=0x38d7d16\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readPrimaryTable(0x14197f8c8370, 0x14197fb50000)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:258 +0xb1 fp=0x14197fcf3ec8 sp=0x14197fcf3e38 pc=0x38d8171\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x14197f8c8370)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:174 +0x189 fp=0x14197fcf3f30 sp=0x14197fcf3ec8 pc=0x38d7b89\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x14197f8c8370, 0x14197f59aee0)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x14197fcf3fc0 sp=0x14197fcf3f30 pc=0x38d788c\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x14197fcf3fe0 sp=0x14197fcf3fc0 pc=0x38d751b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf3fe8 sp=0x14197fcf3fe0 pc=0x1f40801\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 184\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc\n\nrax    0x0\nrbx    0x7f808bfff640\nrcx    0x7f80df60594c\nrdx    0x6\nrdi    0x109fd\nrsi    0x10a00\nrbp    0x10a00\nrsp    0x7f808bffe790\nr8     0x7f808bffe860\nr9     0x7f80df72b4e0\nr10    0x8\nr11    0x246\nr12    0x6\nr13    0x7f808bffeb60\nr14    0x14197f1334a0\nr15    0xffffffffffffffff\nrip    0x7f80df60594c\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\n\n  query-infra-svc: command failed: exit status 2\noutput: runtime/cgo: pthread_create failed: Resource temporarily unavailable\nSIGABRT: abort\nPC=0x7f7ef9baa94c m=3 sigcode=18446744073709551610\n\ngoroutine 0 gp=0x170ac5c1f0e0 m=3 mp=0x170ac5c91808 [idle]:\nruntime: g 0 gp=0x170ac5c1f0e0: unknown pc 0x7f7ef9baa94c\nstack: frame={sp:0x7f7eabffe790, fp:0x0} stack=[0x7f7eab7ff1e0,0x7f7eabffede0)\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n00007f7eabffe690: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6a0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6b0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6c0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6d0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6e0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6f0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe700: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe710: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe720: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe730: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe740: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe750: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe760: 00000000 00000000  00007f7e f9bb7fa9  ............~...\n00007f7eabffe770: 00000000 00000000  00000000 00000011  ................\n00007f7eabffe780: 00000000 00000130  00007f7e f9baa93e  0.......>...~...\n00007f7eabffe790: 00000000 00000120  00000000 00000000   ...............\n                  ^ SP\n00007f7eabffe7a0: 00000000 00000013  00000000 00000000  ................\n00007f7eabffe7b0: 00000000 00000000  00007f7e f9c2dcff  ............~...\n00007f7eabffe7c0: 00007f7e 7bfff910  00007f7e f9c2dcb1  ...{~.......~...\n00007f7eabffe7d0: 00007f7e 7bfff640  00007f7e abffeaa0  @..{~.......~...\n00007f7eabffe7e0: 00007f7e abffe90e  00007f7e abffe90f  ....~.......~...\n00007f7eabffe7f0: 00007f7e 7bfff640  00007f7e f9ba87f5  @..{~.......~...\n00007f7eabffe800: 00007f7e a4000020  00000000 00000120   ...~... .......\n00007f7eabffe810: 00000000 003d0f00  176a34c4 bac87500  ..=......u...4j.\n00007f7eabffe820: 00007f7e abfff640  00000000 00000006  @...~...........\n00007f7eabffe830: 00007f7e a4000f30  00007f7e abffeb60  0...~...`...~...\n00007f7eabffe840: 0000170a c5c1f0e0  00007f7e f9b5d646  ........F...~...\n00007f7eabffe850: 00007f7e f9d1ae90  00007f7e f9b477f3  ....~....w..~...\n00007f7eabffe860: 00000000 00000020  176a34c4 bac87500   ........u...4j.\n00007f7eabffe870: 00007f7e 7b7ff000  00007f7e f9c1cac4  ...{~.......~...\n00007f7eabffe880: 00000000 00000000  00000000 0000000d  ................\nruntime: g 0 gp=0x170ac5c1f0e0: unknown pc 0x7f7ef9baa94c\nstack: frame={sp:0x7f7eabffe790, fp:0x0} stack=[0x7f7eab7ff1e0,0x7f7eabffede0)\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n00007f7eabffe690: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6a0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6b0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6c0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6d0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6e0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe6f0: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe700: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe710: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe720: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe730: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe740: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe750: 00000000 00000000  00000000 00000000  ................\n00007f7eabffe760: 00000000 00000000  00007f7e f9bb7fa9  ............~...\n00007f7eabffe770: 00000000 00000000  00000000 00000011  ................\n00007f7eabffe780: 00000000 00000130  00007f7e f9baa93e  0.......>...~...\n00007f7eabffe790: 00000000 00000120  00000000 00000000   ...............\n                  ^ SP\n00007f7eabffe7a0: 00000000 00000013  00000000 00000000  ................\n00007f7eabffe7b0: 00000000 00000000  00007f7e f9c2dcff  ............~...\n00007f7eabffe7c0: 00007f7e 7bfff910  00007f7e f9c2dcb1  ...{~.......~...\n00007f7eabffe7d0: 00007f7e 7bfff640  00007f7e abffeaa0  @..{~.......~...\n00007f7eabffe7e0: 00007f7e abffe90e  00007f7e abffe90f  ....~.......~...\n00007f7eabffe7f0: 00007f7e 7bfff640  00007f7e f9ba87f5  @..{~.......~...\n00007f7eabffe800: 00007f7e a4000020  00000000 00000120   ...~... .......\n00007f7eabffe810: 00000000 003d0f00  176a34c4 bac87500  ..=......u...4j.\n00007f7eabffe820: 00007f7e abfff640  00000000 00000006  @...~...........\n00007f7eabffe830: 00007f7e a4000f30  00007f7e abffeb60  0...~...`...~...\n00007f7eabffe840: 0000170a c5c1f0e0  00007f7e f9b5d646  ........F...~...\n00007f7eabffe850: 00007f7e f9d1ae90  00007f7e f9b477f3  ....~....w..~...\n00007f7eabffe860: 00000000 00000020  176a34c4 bac87500   ........u...4j.\n00007f7eabffe870: 00007f7e 7b7ff000  00007f7e f9c1cac4  ...{~.......~...\n00007f7eabffe880: 00000000 00000000  00000000 0000000d  ................\n\ngoroutine 1 gp=0x170ac5c1e1e0 m=nil [sync.WaitGroup.Wait]:\nruntime.gopark(0x170ac5cb27b8?, 0x2022a4d?, 0x70?, 0x8e?, 0x170ac5cb27e8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cb27a0 sp=0x170ac5cb2780 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x170ac65f8210, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x170ac5cb2808 sp=0x170ac5cb27a0 pc=0x1f162b2\nsync.runtime_SemacquireWaitGroup(0x170ac6709950?, 0xe0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x170ac5cb2840 sp=0x170ac5cb2808 pc=0x1f3a66e\nsync.(*WaitGroup).Wait(0x170ac65f8208)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x170ac5cb2868 sp=0x170ac5cb2840 pc=0x1f4f445\ngolang.org/x/sync/errgroup.(*Group).Wait(0x170ac65f8200)\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x170ac5cb2888 sp=0x170ac5cb2868 pc=0x362507e\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0}, {0x170ac626a1e0, 0x6, 0x6})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x170ac5cb2948 sp=0x170ac5cb2888 pc=0x3927e05\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).gatherInfraLogs(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:595 +0xadd fp=0x170ac5cb2d30 sp=0x170ac5cb2948 pc=0x392799d\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:333 +0x533 fp=0x170ac5cb3908 sp=0x170ac5cb2d30 pc=0x3921bf3\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedInfraQueryOptions).RunInfra(0x170ac5c2c108, {0x59f0b08, 0x170ac64108d0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:83 +0x71f fp=0x170ac5cb3b30 sp=0x170ac5cb3908 pc=0x393887f\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawInfraQueryOptions).Run(0x0?, {0x59f0b08, 0x170ac64108d0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_options.go:109 +0x54 fp=0x170ac5cb3b58 sp=0x170ac5cb3b30 pc=0x3939134\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryInfraCommand.func1(0x170ac66f5508?, {0x170ac5d40960?, 0x4?, 0x553b0e2?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:45 +0x28 fp=0x170ac5cb3b80 sp=0x170ac5cb3b58 pc=0x3938128\ngithub.com/spf13/cobra.(*Command).execute(0x170ac66f5508, {0x170ac5e36100, 0xa, 0xa})\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x170ac5cb3d28 sp=0x170ac5cb3b80 pc=0x212e614\ngithub.com/spf13/cobra.(*Command).ExecuteC(0x170ac6228608)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x170ac5cb3e18 sp=0x170ac5cb3d28 pc=0x212efa5\ngithub.com/spf13/cobra.(*Command).Execute(...)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071\ngithub.com/spf13/cobra.(*Command).ExecuteContext(...)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064\nmain.main()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x170ac5cb3f48 sp=0x170ac5cb3e18 pc=0x47ddd6a\nruntime.main()\n\t/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x170ac5cb3fe0 sp=0x170ac5cb3f48 pc=0x1f01975\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cb3fe8 sp=0x170ac5cb3fe0 pc=0x1f40801\n\ngoroutine 2 gp=0x170ac5c1ed20 m=nil [force gc (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdefa8 sp=0x170ac5cdef88 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.forcegchelper()\n\t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x170ac5cdefe0 sp=0x170ac5cdefa8 pc=0x1f01c93\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdefe8 sp=0x170ac5cdefe0 pc=0x1f40801\ncreated by runtime.init.7 in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\n\ngoroutine 18 gp=0x170ac5d843c0 m=nil [GC sweep wait]:\nruntime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cda788 sp=0x170ac5cda768 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.bgsweep(0x170ac5d92000)\n\t/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x170ac5cda7c8 sp=0x170ac5cda788 pc=0x1ee9f11\nruntime.gcenable.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x170ac5cda7e0 sp=0x170ac5cda7c8 pc=0x1edadd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cda7e8 sp=0x170ac5cda7e0 pc=0x1f40801\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\n\ngoroutine 19 gp=0x170ac5d845a0 m=nil [GC scavenge wait]:\nruntime.gopark(0x10000?, 0x5981cd8?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdaf78 sp=0x170ac5cdaf58 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.(*scavengerState).park(0x82979c0)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x170ac5cdafa8 sp=0x170ac5cdaf78 pc=0x1ee77e9\nruntime.bgscavenge(0x170ac5d92000)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x170ac5cdafc8 sp=0x170ac5cdafa8 pc=0x1ee7d99\nruntime.gcenable.gowrap2()\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x170ac5cdafe0 sp=0x170ac5cdafc8 pc=0x1edad97\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdafe8 sp=0x170ac5cdafe0 pc=0x1f40801\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\n\ngoroutine 34 gp=0x170ac5e00000 m=nil [GOMAXPROCS updater (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e06788 sp=0x170ac5e06768 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.updateMaxProcsGoroutine()\n\t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x170ac5e067e0 sp=0x170ac5e06788 pc=0x1f10267\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e067e8 sp=0x170ac5e067e0 pc=0x1f40801\ncreated by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\n\ngoroutine 35 gp=0x170ac5e005a0 m=nil [finalizer wait]:\nruntime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x170ac5cde601?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cde620 sp=0x170ac5cde600 pc=0x1f38d2e\nruntime.runFinalizers()\n\t/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x170ac5cde7e0 sp=0x170ac5cde620 pc=0x1ed9cc7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cde7e8 sp=0x170ac5cde7e0 pc=0x1f40801\ncreated by runtime.createfing in goroutine 1\n\t/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d\n\ngoroutine 36 gp=0x170ac5ff81e0 m=nil [cleanup wait]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e06f68 sp=0x170ac5e06f48 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.(*cleanupQueue).dequeue(0x82980e0)\n\t/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x170ac5e06fa0 sp=0x170ac5e06f68 pc=0x1ed6074\nruntime.runCleanups()\n\t/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x170ac5e06fe0 sp=0x170ac5e06fa0 pc=0x1ed66e5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e06fe8 sp=0x170ac5e06fe0 pc=0x1f40801\ncreated by runtime.(*cleanupQueue).createGs in goroutine 1\n\t/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5\n\ngoroutine 37 gp=0x170ac611bc20 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e07740 sp=0x170ac5e07720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5e077c8 sp=0x170ac5e07740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5e077e0 sp=0x170ac5e077c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e077e8 sp=0x170ac5e077e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 3 gp=0x170ac5c1f680 m=nil [runnable]:\nruntime.gopark(0x125c26559f5b?, 0x3?, 0x94?, 0x85?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdf740 sp=0x170ac5cdf720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdf7c8 sp=0x170ac5cdf740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdf7e0 sp=0x170ac5cdf7c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdf7e8 sp=0x170ac5cdf7e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 20 gp=0x170ac5d84780 m=nil [GC worker (idle)]:\nruntime.gopark(0x125c265c3db1?, 0x3?, 0x92?, 0xcd?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdb740 sp=0x170ac5cdb720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdb7c8 sp=0x170ac5cdb740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdb7e0 sp=0x170ac5cdb7c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdb7e8 sp=0x170ac5cdb7e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 38 gp=0x170ac6134000 m=nil [GC worker (idle)]:\nruntime.gopark(0x125c26588b9e?, 0x1?, 0xf7?, 0xb4?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e07f40 sp=0x170ac5e07f20 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5e07fc8 sp=0x170ac5e07f40 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5e07fe0 sp=0x170ac5e07fc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e07fe8 sp=0x170ac5e07fe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 4 gp=0x170ac5c1f860 m=10 mp=0x170ac6280008 [GC mark termination]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x170ac5cdfea0 sp=0x170ac5cdfe90 pc=0x1f3ebc8\nruntime.forEachP(...)\n\t/usr/lib/golang/src/runtime/proc.go:2112\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x170ac5cdff40 sp=0x170ac5cdfea0 pc=0x1edbc34\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x170ac5cdffc8 sp=0x170ac5cdff40 pc=0x1edda88\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdffe0 sp=0x170ac5cdffc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdffe8 sp=0x170ac5cdffe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 21 gp=0x170ac5d84960 m=nil [GC worker (idle)]:\nruntime.gopark(0x125c265818cf?, 0x3?, 0xa5?, 0x9c?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdbf40 sp=0x170ac5cdbf20 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdbfc8 sp=0x170ac5cdbf40 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdbfe0 sp=0x170ac5cdbfc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdbfe8 sp=0x170ac5cdbfe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 22 gp=0x170ac5d84b40 m=nil [GC worker (idle)]:\nruntime.gopark(0x125c265846e7?, 0x3?, 0x77?, 0xb6?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdc740 sp=0x170ac5cdc720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdc7c8 sp=0x170ac5cdc740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdc7e0 sp=0x170ac5cdc7c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdc7e8 sp=0x170ac5cdc7e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 23 gp=0x170ac5d84d20 m=nil [GC worker (idle)]:\nruntime.gopark(0x125c26581951?, 0x3?, 0x91?, 0x58?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdcf40 sp=0x170ac5cdcf20 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdcfc8 sp=0x170ac5cdcf40 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdcfe0 sp=0x170ac5cdcfc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdcfe8 sp=0x170ac5cdcfe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 39 gp=0x170ac648a3c0 m=nil [select, locked to thread]:\nruntime.gopark(0x170ac5e097a8?, 0x2?, 0x1?, 0xa3?, 0x170ac5e09794?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e09628 sp=0x170ac5e09608 pc=0x1f38d2e\nruntime.selectgo(0x170ac5e097a8, 0x170ac5e09790, 0x0?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5e09758 sp=0x170ac5e09628 pc=0x1f152a5\nruntime.ensureSigM.func1()\n\t/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x170ac5e097e0 sp=0x170ac5e09758 pc=0x1f328e8\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e097e8 sp=0x170ac5e097e0 pc=0x1f40801\ncreated by runtime.ensureSigM in goroutine 1\n\t/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5\n\ngoroutine 24 gp=0x170ac62f4960 m=8 mp=0x170ac6180008 [syscall]:\nruntime.notetsleepg(0x82bdea0, 0xffffffffffffffff)\n\t/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x170ac5e037a0 sp=0x170ac5e03778 pc=0x1ece509\nos/signal.signal_recv()\n\t/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x170ac5e037c0 sp=0x170ac5e037a0 pc=0x1f3b298\nos/signal.loop()\n\t/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x170ac5e037e0 sp=0x170ac5e037c0 pc=0x2030553\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e037e8 sp=0x170ac5e037e0 pc=0x1f40801\ncreated by os/signal.Notify.func1.1 in goroutine 1\n\t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\n\ngoroutine 50 gp=0x170ac6135a40 m=nil [select]:\nruntime.gopark(0x170ac66e3798?, 0x2?, 0x80?, 0x6f?, 0x170ac66e378c?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac66e3628 sp=0x170ac66e3608 pc=0x1f38d2e\nruntime.selectgo(0x170ac66e3798, 0x170ac66e3788, 0x0?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac66e3758 sp=0x170ac66e3628 pc=0x1f152a5\nos/signal.NotifyContext.func1()\n\t/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x170ac66e37e0 sp=0x170ac66e3758 pc=0x202ff9c\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac66e37e8 sp=0x170ac66e37e0 pc=0x1f40801\ncreated by os/signal.NotifyContext in goroutine 1\n\t/usr/lib/golang/src/os/signal/signal.go:291 +0x158\n\ngoroutine 102 gp=0x170ac61843c0 m=nil [select]:\nruntime.gopark(0x170ac5cefc70?, 0x2?, 0xc8?, 0xf9?, 0x170ac5cefb6c?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cef9f0 sp=0x170ac5cef9d0 pc=0x1f38d2e\nruntime.selectgo(0x170ac5cefc70, 0x170ac5cefb68, 0x554857c?, 0x0, 0x226?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5cefb20 sp=0x170ac5cef9f0 pc=0x1f152a5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x170ac5c49130}, 0x170ac65fa070, 0x170ac6410cf0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:283 +0x21b fp=0x170ac5ceff48 sp=0x170ac5cefb20 pc=0x39204fb\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x170ac5ceff78 sp=0x170ac5ceff48 pc=0x392800c\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5ceffe0 sp=0x170ac5ceff78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5ceffe8 sp=0x170ac5ceffe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 104 gp=0x170ac6184d20 m=nil [select]:\nruntime.gopark(0x170ac6992ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69929d8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6992848 sp=0x170ac6992828 pc=0x1f38d2e\nruntime.selectgo(0x170ac6992ac8, 0x170ac69929d0, 0x55562e1?, 0x0, 0x1f6d6cf?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6992978 sp=0x170ac6992848 pc=0x1f152a5\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac619fb80, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac6992b40 sp=0x170ac6992978 pc=0x22d92e5\nnet/http.(*http2ClientConn).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac619fb80, {0x4c?, 0xfa?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac6992c80 sp=0x170ac6992b40 pc=0x22d50ad\nnet/http.(*http2Transport).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac619fb80?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac6992cc0 sp=0x170ac6992c80 pc=0x22e3a9c\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac619fa40)\n\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac6992ea0 sp=0x170ac6992cc0 pc=0x231037f\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac6992ec0 sp=0x170ac6992ea0 pc=0x2321178\nnet/http.send(0x170ac619fa40, {0x59a1ce0, 0x8175400}, {0x170ac6993138?, 0x1f3bac6?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69930b0 sp=0x170ac6992ec0 pc=0x22ad9ab\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac619fa40, {0x170ac69931e8?, 0x1fdd787?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac6993148 sp=0x170ac69930b0 pc=0x22ad1d8\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac619fa40)\n\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac6993338 sp=0x170ac6993148 pc=0x22af817\nnet/http.(*Client).Do(...)\n\t/usr/lib/golang/src/net/http/client.go:592\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5c49450}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e34a0}, 0x170ac65f7ce0, {0x170ac65d9560, 0x112})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac6993630 sp=0x170ac6993338 pc=0x38de6bd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5c49450}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x170ac65f7c80, 0x0, {0x0, ...}, ...})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac6993888 sp=0x170ac6993630 pc=0x38ddecd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5c49450?}, 0xa8?, {0x170ac5db1a50?, 0x170ac6764a48?}, 0x3?, 0x2022a4d?)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac6993960 sp=0x170ac6993888 pc=0x38dd40b\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194700?}, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x170ac6764a48, 0x3, 0x4})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69939e0 sp=0x170ac6993960 pc=0x38e51e2\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194700}, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x0?, 0x2?, 0x1?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac6993a78 sp=0x170ac69939e0 pc=0x38e4eaf\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020500}, 0x170ac65fa000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac6993d60 sp=0x170ac6993a78 pc=0x391b5c5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac5fe1880?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020500?}, 0x0?)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac6993e88 sp=0x170ac6993d60 pc=0x3929c16\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac6993f78 sp=0x170ac6993e88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac6993fe0 sp=0x170ac6993f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6993fe8 sp=0x170ac6993fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 101 gp=0x170ac61d4000 m=nil [sync.WaitGroup.Wait]:\nruntime.gopark(0x170ac5de7e00?, 0x1ed0765?, 0xb0?, 0xfa?, 0xe?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de7da0 sp=0x170ac5de7d80 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x170ac65f8290, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x170ac5de7e08 sp=0x170ac5de7da0 pc=0x1f162b2\nsync.runtime_SemacquireWaitGroup(0x170ac6709a70?, 0x0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x170ac5de7e40 sp=0x170ac5de7e08 pc=0x1f3a66e\nsync.(*WaitGroup).Wait(0x170ac65f8288)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x170ac5de7e68 sp=0x170ac5de7e40 pc=0x1f4f445\ngolang.org/x/sync/errgroup.(*Group).Wait(0x170ac65f8280)\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x170ac5de7e88 sp=0x170ac5de7e68 pc=0x362507e\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x170ac5fe1140, {0x59f0b40, 0x170ac5c49130}, {0x170ac626a1e0, 0x6, 0x170ac5cac1c0?}, 0x170ac65fa000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x170ac5de7f00 sp=0x170ac5de7e88 pc=0x392976a\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x170ac5de7f78 sp=0x170ac5de7f00 pc=0x392808e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5de7fe0 sp=0x170ac5de7f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de7fe8 sp=0x170ac5de7fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 67 gp=0x170ac62f5a40 m=nil [IO wait]:\nruntime.gopark(0x170ac6497780?, 0x8?, 0x40?, 0x16?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de47b8 sp=0x170ac5de4798 pc=0x1f38d2e\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x170ac5de47f0 sp=0x170ac5de47b8 pc=0x1ef9e57\ninternal/poll.runtime_pollWait(0x7f7eb1670000, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x170ac5de4810 sp=0x170ac5de47f0 pc=0x1f375c5\ninternal/poll.(*pollDesc).wait(0x170ac620b480?, 0x170ac65a2000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x170ac5de4838 sp=0x170ac5de4810 pc=0x1fbd167\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0x170ac620b480, {0x170ac65a2000, 0x3500, 0x3500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x170ac5de48d0 sp=0x170ac5de4838 pc=0x1fbe4ce\nnet.(*netFD).Read(0x170ac620b480, {0x170ac65a2000?, 0x170ac5de4901?, 0x22a2e75?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x170ac5de4918 sp=0x170ac5de48d0 pc=0x20798c5\nnet.(*conn).Read(0x170ac62823d8, {0x170ac65a2000?, 0x7f7eb047fc00?, 0x7f7ef9af2e00?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x170ac5de4960 sp=0x170ac5de4918 pc=0x208b7a5\ncrypto/tls.(*atLeastReader).Read(0x170ac6708f30, {0x170ac65a2000?, 0x170ac5de4a48?, 0x1efcc28?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x170ac5de49a8 sp=0x170ac5de4960 pc=0x2222ddb\nbytes.(*Buffer).ReadFrom(0x170ac605fb28, {0x59a3680, 0x170ac6708f30})\n\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x170ac5de4a00 sp=0x170ac5de49a8 pc=0x1fe9258\ncrypto/tls.(*Conn).readFromUntil(0x170ac605f888, {0x59a32c0, 0x170ac62823d8}, 0x222764c?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x170ac5de4a38 sp=0x170ac5de4a00 pc=0x2222fbe\ncrypto/tls.(*Conn).readRecordOrCCS(0x170ac605f888, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x170ac5de4c18 sp=0x170ac5de4a38 pc=0x222009b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:588\ncrypto/tls.(*Conn).Read(0x170ac605f888, {0x170ac6029000, 0x1000, 0x22c34e0?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x170ac5de4c80 sp=0x170ac5de4c18 pc=0x22267c5\nbufio.(*Reader).Read(0x170ac647aa20, {0x170ac6153624, 0x9, 0x22e100e?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0x170ac5de4cb8 sp=0x170ac5de4c80 pc=0x20a71f7\nio.ReadAtLeast({0x59a1260, 0x170ac647aa20}, {0x170ac6153624, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0x170ac5de4d00 sp=0x170ac5de4cb8 pc=0x1fb17ee\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\nnet/http.http2readFrameHeader({0x170ac6153624, 0x9, 0x170a00000005?}, {0x59a1260?, 0x170ac647aa20?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:1805 +0x65 fp=0x170ac5de4d50 sp=0x170ac5de4d00 pc=0x22b9e25\nnet/http.(*http2Framer).ReadFrameHeader(0x170ac61535e0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:2071 +0x6b fp=0x170ac5de4dd8 sp=0x170ac5de4d50 pc=0x22ba9eb\nnet/http.(*http2Framer).ReadFrame(0x170ac61535e0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:2130 +0x18 fp=0x170ac5de4e18 sp=0x170ac5de4dd8 pc=0x22bb1b8\nnet/http.(*http2clientConnReadLoop).run(0x170ac5de4fa8)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9550 +0xca fp=0x170ac5de4f60 sp=0x170ac5de4e18 pc=0x22de14a\nnet/http.(*http2ClientConn).readLoop(0x170ac61d9880)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x170ac5de4fc8 sp=0x170ac5de4f60 pc=0x22dd532\nnet/http.(*http2Transport).newClientConn.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x170ac5de4fe0 sp=0x170ac5de4fc8 pc=0x22d7057\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de4fe8 sp=0x170ac5de4fe0 pc=0x1f40801\ncreated by net/http.(*http2Transport).newClientConn in goroutine 66\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5\n\ngoroutine 73 gp=0x170ac648ad20 m=nil [IO wait]:\nruntime.gopark(0x170ac6497e40?, 0x1edb176?, 0x80?, 0xed?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de6808 sp=0x170ac5de67e8 pc=0x1f38d2e\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x170ac5de6840 sp=0x170ac5de6808 pc=0x1ef9e57\ninternal/poll.runtime_pollWait(0x7f7eb166fe00, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x170ac5de6860 sp=0x170ac5de6840 pc=0x1f375c5\ninternal/poll.(*pollDesc).wait(0x170ac60c4e80?, 0x170ac5ecd300?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x170ac5de6888 sp=0x170ac5de6860 pc=0x1fbd167\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0x170ac60c4e80, {0x170ac5ecd300, 0x1300, 0x1300})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x170ac5de6920 sp=0x170ac5de6888 pc=0x1fbe4ce\nnet.(*netFD).Read(0x170ac60c4e80, {0x170ac5ecd300?, 0x170ac5ecd30a?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x170ac5de6968 sp=0x170ac5de6920 pc=0x20798c5\nnet.(*conn).Read(0x170ac5d82278, {0x170ac5ecd300?, 0x7f7eb047fc00?, 0x7f7ef9af2e00?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x170ac5de69b0 sp=0x170ac5de6968 pc=0x208b7a5\ncrypto/tls.(*atLeastReader).Read(0x170ac67088e8, {0x170ac5ecd300?, 0x170ac5de6a98?, 0x1efcc28?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x170ac5de69f8 sp=0x170ac5de69b0 pc=0x2222ddb\nbytes.(*Buffer).ReadFrom(0x170ac5dd02a8, {0x59a3680, 0x170ac67088e8})\n\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x170ac5de6a50 sp=0x170ac5de69f8 pc=0x1fe9258\ncrypto/tls.(*Conn).readFromUntil(0x170ac5dd0008, {0x59a32c0, 0x170ac5d82278}, 0x222764c?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x170ac5de6a88 sp=0x170ac5de6a50 pc=0x2222fbe\ncrypto/tls.(*Conn).readRecordOrCCS(0x170ac5dd0008, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x170ac5de6c68 sp=0x170ac5de6a88 pc=0x222009b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:588\ncrypto/tls.(*Conn).Read(0x170ac5dd0008, {0x170ac6564000, 0x1000, 0x4e55540?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x170ac5de6cd0 sp=0x170ac5de6c68 pc=0x22267c5\nnet/http.(*persistConn).Read(0x170ac5f73180, {0x170ac6564000?, 0x59a0ea0?, 0x81260e0?})\n\t/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x170ac5de6d30 sp=0x170ac5de6cd0 pc=0x2319d07\nbufio.(*Reader).fill(0x170ac65537a0)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x170ac5de6d68 sp=0x170ac5de6d30 pc=0x20a6c63\nbufio.(*Reader).Peek(0x170ac65537a0, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x170ac5de6d80 sp=0x170ac5de6d68 pc=0x20a6df3\nnet/http.(*persistConn).readLoop(0x170ac5f73180)\n\t/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x170ac5de6fc8 sp=0x170ac5de6d80 pc=0x231a812\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x170ac5de6fe0 sp=0x170ac5de6fc8 pc=0x2319197\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de6fe8 sp=0x170ac5de6fe0 pc=0x1f40801\ncreated by net/http.(*Transport).dialConn in goroutine 82\n\t/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac\n\ngoroutine 103 gp=0x170ac648af00 m=nil [select]:\nruntime.gopark(0x170ac5de0d00?, 0x2?, 0xa0?, 0x6a?, 0x170ac5de0c74?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de0ac8 sp=0x170ac5de0aa8 pc=0x1f38d2e\nruntime.selectgo(0x170ac5de0d00, 0x170ac5de0c70, 0x553bfdc?, 0x0, 0x5a48c80?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5de0bf8 sp=0x170ac5de0ac8 pc=0x1f152a5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x0?, {0x59f0b40, 0x170ac5c49130}, 0x170ac65fa000, 0x170ac65fa070)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x170ac5de0f10 sp=0x170ac5de0bf8 pc=0x392836e\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x170ac5de0f78 sp=0x170ac5de0f10 pc=0x3927f3d\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5de0fe0 sp=0x170ac5de0f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de0fe8 sp=0x170ac5de0fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 74 gp=0x170ac648b4a0 m=nil [select]:\nruntime.gopark(0x170ac5df1f48?, 0x2?, 0x0?, 0xb6?, 0x170ac5df1ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6760d88 sp=0x170ac6760d68 pc=0x1f38d2e\nruntime.selectgo(0x170ac6760f48, 0x170ac5df1ef0, 0x170ac68d6980?, 0x0, 0x170ac655e8d0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6760eb8 sp=0x170ac6760d88 pc=0x1f152a5\nnet/http.(*persistConn).writeLoop(0x170ac5f73180)\n\t/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x170ac6760fc8 sp=0x170ac6760eb8 pc=0x231c426\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x170ac6760fe0 sp=0x170ac6760fc8 pc=0x2319157\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6760fe8 sp=0x170ac6760fe0 pc=0x1f40801\ncreated by net/http.(*Transport).dialConn in goroutine 82\n\t/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe\n\ngoroutine 105 gp=0x170ac61d43c0 m=nil [select]:\nruntime.gopark(0x170ac699cac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac699c9d8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac699c848 sp=0x170ac699c828 pc=0x1f38d2e\nruntime.selectgo(0x170ac699cac8, 0x170ac699c9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac699c978 sp=0x170ac699c848 pc=0x1f152a5\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e000, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac699cb40 sp=0x170ac699c978 pc=0x22d92e5\nnet/http.(*http2ClientConn).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e000, {0x4c?, 0xfa?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac699cc80 sp=0x170ac699cb40 pc=0x22d50ad\nnet/http.(*http2Transport).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e000?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac699ccc0 sp=0x170ac699cc80 pc=0x22e3a9c\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac619fcc0)\n\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac699cea0 sp=0x170ac699ccc0 pc=0x231037f\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac699cec0 sp=0x170ac699cea0 pc=0x2321178\nnet/http.send(0x170ac619fcc0, {0x59a1ce0, 0x8175400}, {0x170ac699d138?, 0x1f3bac6?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac699d0b0 sp=0x170ac699cec0 pc=0x22ad9ab\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac619fcc0, {0x170ac699d1e8?, 0x1fdd787?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac699d148 sp=0x170ac699d0b0 pc=0x22ad1d8\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac619fcc0)\n\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac699d338 sp=0x170ac699d148 pc=0x22af817\nnet/http.(*Client).Do(...)\n\t/usr/lib/golang/src/net/http/client.go:592\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e000}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e36b0}, 0x170ac65f7e00, {0x170ac65d9680, 0x118})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac699d630 sp=0x170ac699d338 pc=0x38de6bd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e000}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x170ac65f7da0, 0x0, {0x0, ...}, ...})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac699d888 sp=0x170ac699d630 pc=0x38ddecd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e000?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de1a48?}, 0x3?, 0x2022a4d?)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac699d960 sp=0x170ac699d888 pc=0x38dd40b\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194850?}, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x170ac5de1a48, 0x3, 0x4})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac699d9e0 sp=0x170ac699d960 pc=0x38e51e2\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194850}, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x0?, 0x2?, 0x0?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac699da78 sp=0x170ac699d9e0 pc=0x38e4eaf\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac60207c0}, 0x170ac65fa000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac699dd60 sp=0x170ac699da78 pc=0x391b5c5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac60207c0?}, 0x0?)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac699de88 sp=0x170ac699dd60 pc=0x3929c16\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac699df78 sp=0x170ac699de88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac699dfe0 sp=0x170ac699df78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac699dfe8 sp=0x170ac699dfe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 107 gp=0x170ac61d45a0 m=nil [select]:\nruntime.gopark(0x170ac69b2ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69b29d8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69b2848 sp=0x170ac69b2828 pc=0x1f38d2e\nruntime.selectgo(0x170ac69b2ac8, 0x170ac69b29d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69b2978 sp=0x170ac69b2848 pc=0x1f152a5\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e500, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69b2b40 sp=0x170ac69b2978 pc=0x22d92e5\nnet/http.(*http2ClientConn).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e500, {0x4c?, 0xfa?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69b2c80 sp=0x170ac69b2b40 pc=0x22d50ad\nnet/http.(*http2Transport).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e500?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69b2cc0 sp=0x170ac69b2c80 pc=0x22e3a9c\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac699e3c0)\n\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69b2ea0 sp=0x170ac69b2cc0 pc=0x231037f\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69b2ec0 sp=0x170ac69b2ea0 pc=0x2321178\nnet/http.send(0x170ac699e3c0, {0x59a1ce0, 0x8175400}, {0x170ac69b3138?, 0x1f3bac6?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69b30b0 sp=0x170ac69b2ec0 pc=0x22ad9ab\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac699e3c0, {0x4d3aa8?, 0xcd6610?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69b3148 sp=0x170ac69b30b0 pc=0x22ad1d8\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac699e3c0)\n\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69b3338 sp=0x170ac69b3148 pc=0x22af817\nnet/http.(*Client).Do(...)\n\t/usr/lib/golang/src/net/http/client.go:592\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e1e0}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3a60}, 0x170ac69a0120, {0x170ac65d98c0, 0x111})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69b3630 sp=0x170ac69b3338 pc=0x38de6bd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e1e0}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x170ac69a00c0, 0x0, {0x0, ...}, ...})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69b3888 sp=0x170ac69b3630 pc=0x38ddecd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e1e0?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de2a48?}, 0x3?, 0x2022a4d?)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69b3960 sp=0x170ac69b3888 pc=0x38dd40b\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194bd0?}, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x170ac5de2a48, 0x3, 0x4})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69b39e0 sp=0x170ac69b3960 pc=0x38e51e2\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194bd0}, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x0?, 0x2?, 0x0?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69b3a78 sp=0x170ac69b39e0 pc=0x38e4eaf\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020ec0}, 0x170ac65fa000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69b3d60 sp=0x170ac69b3a78 pc=0x391b5c5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020ec0?}, 0x0?)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69b3e88 sp=0x170ac69b3d60 pc=0x3929c16\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69b3f78 sp=0x170ac69b3e88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69b3fe0 sp=0x170ac69b3f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69b3fe8 sp=0x170ac69b3fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 106 gp=0x170ac648b680 m=nil [select]:\nruntime.gopark(0x170ac69aaac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69aa9d8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69aa848 sp=0x170ac69aa828 pc=0x1f38d2e\nruntime.selectgo(0x170ac69aaac8, 0x170ac69aa9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69aa978 sp=0x170ac69aa848 pc=0x1f152a5\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e280, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69aab40 sp=0x170ac69aa978 pc=0x22d92e5\nnet/http.(*http2ClientConn).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e280, {0x4c?, 0xfa?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69aac80 sp=0x170ac69aab40 pc=0x22d50ad\nnet/http.(*http2Transport).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e280?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69aacc0 sp=0x170ac69aac80 pc=0x22e3a9c\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac699e140)\n\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69aaea0 sp=0x170ac69aacc0 pc=0x231037f\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69aaec0 sp=0x170ac69aaea0 pc=0x2321178\nnet/http.send(0x170ac699e140, {0x59a1ce0, 0x8175400}, {0x170ac69ab138?, 0x1f3bac6?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69ab0b0 sp=0x170ac69aaec0 pc=0x22ad9ab\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac699e140, {0x170ac69ab1e8?, 0x1fdd787?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69ab148 sp=0x170ac69ab0b0 pc=0x22ad1d8\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac699e140)\n\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69ab338 sp=0x170ac69ab148 pc=0x22af817\nnet/http.(*Client).Do(...)\n\t/usr/lib/golang/src/net/http/client.go:592\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e0f0}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e38c0}, 0x170ac69a0030, {0x170ac65d97a0, 0x10e})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69ab630 sp=0x170ac69ab338 pc=0x38de6bd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e0f0}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x170ac65f7ec0, 0x0, {0x0, ...}, ...})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69ab888 sp=0x170ac69ab630 pc=0x38ddecd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e0f0?}, 0xa8?, {0x170ac5db1a50?, 0x170ac6766a48?}, 0x3?, 0x2022a4d?)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69ab960 sp=0x170ac69ab888 pc=0x38dd40b\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194a80?}, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x170ac6766a48, 0x3, 0x4})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69ab9e0 sp=0x170ac69ab960 pc=0x38e51e2\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194a80}, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x0?, 0x2?, 0x0?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69aba78 sp=0x170ac69ab9e0 pc=0x38e4eaf\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020cc0}, 0x170ac65fa000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69abd60 sp=0x170ac69aba78 pc=0x391b5c5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac6766ea0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020cc0?}, 0x170ac5f6f3b0?)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69abe88 sp=0x170ac69abd60 pc=0x3929c16\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69abf78 sp=0x170ac69abe88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69abfe0 sp=0x170ac69abf78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69abfe8 sp=0x170ac69abfe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 108 gp=0x170ac61d4780 m=nil [select]:\nruntime.gopark(0x170ac69c0ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69c09d8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69c0848 sp=0x170ac69c0828 pc=0x1f38d2e\nruntime.selectgo(0x170ac69c0ac8, 0x170ac69c09d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69c0978 sp=0x170ac69c0848 pc=0x1f152a5\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e780, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69c0b40 sp=0x170ac69c0978 pc=0x22d92e5\nnet/http.(*http2ClientConn).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e780, {0x4c?, 0xfa?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69c0c80 sp=0x170ac69c0b40 pc=0x22d50ad\nnet/http.(*http2Transport).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e780?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69c0cc0 sp=0x170ac69c0c80 pc=0x22e3a9c\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac699e640)\n\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69c0ea0 sp=0x170ac69c0cc0 pc=0x231037f\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69c0ec0 sp=0x170ac69c0ea0 pc=0x2321178\nnet/http.send(0x170ac699e640, {0x59a1ce0, 0x8175400}, {0x170ac69c1138?, 0x1f3bac6?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69c10b0 sp=0x170ac69c0ec0 pc=0x22ad9ab\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac699e640, {0x82bed80?, 0x6401fdd700?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69c1148 sp=0x170ac69c10b0 pc=0x22ad1d8\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac699e640)\n\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69c1338 sp=0x170ac69c1148 pc=0x22af817\nnet/http.(*Client).Do(...)\n\t/usr/lib/golang/src/net/http/client.go:592\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e280}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3bf0}, 0x170ac69a0210, {0x170ac65d99e0, 0x110})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69c1630 sp=0x170ac69c1338 pc=0x38de6bd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e280}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x170ac69a01b0, 0x0, {0x0, ...}, ...})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69c1888 sp=0x170ac69c1630 pc=0x38ddecd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e280?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de2a48?}, 0x3?, 0x2022a4d?)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69c1960 sp=0x170ac69c1888 pc=0x38dd40b\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194d90?}, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x170ac5de2a48, 0x3, 0x4})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69c19e0 sp=0x170ac69c1960 pc=0x38e51e2\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194d90}, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x0?, 0x2?, 0x0?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69c1a78 sp=0x170ac69c19e0 pc=0x38e4eaf\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6021d80}, 0x170ac65fa000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69c1d60 sp=0x170ac69c1a78 pc=0x391b5c5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac66e1d6a?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6021d80?}, 0x0?)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69c1e88 sp=0x170ac69c1d60 pc=0x3929c16\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69c1f78 sp=0x170ac69c1e88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69c1fe0 sp=0x170ac69c1f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69c1fe8 sp=0x170ac69c1fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 109 gp=0x170ac61d4960 m=nil [select]:\nruntime.gopark(0x170ac69caac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69ca9d8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69ca848 sp=0x170ac69ca828 pc=0x1f38d2e\nruntime.selectgo(0x170ac69caac8, 0x170ac69ca9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69ca978 sp=0x170ac69ca848 pc=0x1f152a5\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699ea00, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69cab40 sp=0x170ac69ca978 pc=0x22d92e5\nnet/http.(*http2ClientConn).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699ea00, {0x4c?, 0xfa?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69cac80 sp=0x170ac69cab40 pc=0x22d50ad\nnet/http.(*http2Transport).RoundTrip(...)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699ea00?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69cacc0 sp=0x170ac69cac80 pc=0x22e3a9c\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac699e8c0)\n\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69caea0 sp=0x170ac69cacc0 pc=0x231037f\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69caec0 sp=0x170ac69caea0 pc=0x2321178\nnet/http.send(0x170ac699e8c0, {0x59a1ce0, 0x8175400}, {0x170ac69cb138?, 0x1f3bac6?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69cb0b0 sp=0x170ac69caec0 pc=0x22ad9ab\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac699e8c0, {0x170ac69cb1e8?, 0x1fdd787?, 0x0?})\n\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69cb148 sp=0x170ac69cb0b0 pc=0x22ad1d8\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac699e8c0)\n\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69cb338 sp=0x170ac69cb148 pc=0x22af817\nnet/http.(*Client).Do(...)\n\t/usr/lib/golang/src/net/http/client.go:592\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e410}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3d80}, 0x170ac69a0300, {0x170ac65d9b00, 0x114})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69cb630 sp=0x170ac69cb338 pc=0x38de6bd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e410}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x170ac69a02a0, 0x0, {0x0, ...}, ...})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69cb888 sp=0x170ac69cb630 pc=0x38ddecd\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e410?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de3a48?}, 0x3?, 0x2022a4d?)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69cb960 sp=0x170ac69cb888 pc=0x38dd40b\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194620?}, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x170ac5de3a48, 0x3, 0x4})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69cb9e0 sp=0x170ac69cb960 pc=0x38e51e2\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194620}, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x0?, 0x2?, 0x2700000000000000?})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69cba78 sp=0x170ac69cb9e0 pc=0x38e4eaf\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac65f81c0}, 0x170ac65fa000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69cbd60 sp=0x170ac69cba78 pc=0x391b5c5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac65f81c0?}, 0x0?)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69cbe88 sp=0x170ac69cbd60 pc=0x3929c16\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69cbf78 sp=0x170ac69cbe88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69cbfe0 sp=0x170ac69cbf78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69cbfe8 sp=0x170ac69cbfe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 110 gp=0x170ac61d4b40 m=nil [select]:\nruntime.gopark(0x170ac6927f08?, 0x6?, 0x40?, 0x4b?, 0x170ac6927da4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6927c08 sp=0x170ac6927be8 pc=0x1f38d2e\nruntime.selectgo(0x170ac6927f08, 0x170ac6927d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6927d38 sp=0x170ac6927c08 pc=0x1f152a5\nnet/http.(*http2clientStream).writeRequest(0x170ac61a0f00, 0x170ac619fb80, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6927f90 sp=0x170ac6927d38 pc=0x22da6a8\nnet/http.(*http2clientStream).doRequest(0x170ac61a0f00, 0x0?, 0x170ac66e2798?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6927fb8 sp=0x170ac6927f90 pc=0x22d99d8\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6927fe0 sp=0x170ac6927fb8 pc=0x22d999f\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6927fe8 sp=0x170ac6927fe0 pc=0x1f40801\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 104\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n\ngoroutine 111 gp=0x170ac61d4d20 m=nil [select]:\nruntime.gopark(0x170ac658df08?, 0x6?, 0x0?, 0x7?, 0x170ac658dda4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac658dc08 sp=0x170ac658dbe8 pc=0x1f38d2e\nruntime.selectgo(0x170ac658df08, 0x170ac658dd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac658dd38 sp=0x170ac658dc08 pc=0x1f152a5\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1080, 0x170ac699e000, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac658df90 sp=0x170ac658dd38 pc=0x22da6a8\nnet/http.(*http2clientStream).doRequest(0x170ac61a1080, 0x0?, 0x170ac66e2798?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac658dfb8 sp=0x170ac658df90 pc=0x22d99d8\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac658dfe0 sp=0x170ac658dfb8 pc=0x22d999f\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac658dfe8 sp=0x170ac658dfe0 pc=0x1f40801\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 105\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n\ngoroutine 112 gp=0x170ac61d4f00 m=nil [select]:\nruntime.gopark(0x170ac6589f08?, 0x6?, 0x0?, 0x7?, 0x170ac6589da4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6589c08 sp=0x170ac6589be8 pc=0x1f38d2e\nruntime.selectgo(0x170ac6589f08, 0x170ac6589d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6589d38 sp=0x170ac6589c08 pc=0x1f152a5\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1200, 0x170ac699e280, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6589f90 sp=0x170ac6589d38 pc=0x22da6a8\nnet/http.(*http2clientStream).doRequest(0x170ac61a1200, 0x0?, 0x170ac66e1f98?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6589fb8 sp=0x170ac6589f90 pc=0x22d99d8\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6589fe0 sp=0x170ac6589fb8 pc=0x22d999f\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6589fe8 sp=0x170ac6589fe0 pc=0x1f40801\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 106\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n\ngoroutine 113 gp=0x170ac61d50e0 m=nil [select]:\nruntime.gopark(0x170ac658ff08?, 0x6?, 0x0?, 0x7?, 0x170ac658fda4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac658fc08 sp=0x170ac658fbe8 pc=0x1f38d2e\nruntime.selectgo(0x170ac658ff08, 0x170ac658fd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac658fd38 sp=0x170ac658fc08 pc=0x1f152a5\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1380, 0x170ac699e780, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac658ff90 sp=0x170ac658fd38 pc=0x22da6a8\nnet/http.(*http2clientStream).doRequest(0x170ac61a1380, 0x0?, 0x170ac66e1f98?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac658ffb8 sp=0x170ac658ff90 pc=0x22d99d8\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac658ffe0 sp=0x170ac658ffb8 pc=0x22d999f\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac658ffe8 sp=0x170ac658ffe0 pc=0x1f40801\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 108\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n\ngoroutine 114 gp=0x170ac61d52c0 m=nil [select]:\nruntime.gopark(0x170ac6923f08?, 0x6?, 0x0?, 0x7?, 0x170ac6923da4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6923c08 sp=0x170ac6923be8 pc=0x1f38d2e\nruntime.selectgo(0x170ac6923f08, 0x170ac6923d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6923d38 sp=0x170ac6923c08 pc=0x1f152a5\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1500, 0x170ac699ea00, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6923f90 sp=0x170ac6923d38 pc=0x22da6a8\nnet/http.(*http2clientStream).doRequest(0x170ac61a1500, 0x0?, 0x170ac66e1f98?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6923fb8 sp=0x170ac6923f90 pc=0x22d99d8\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6923fe0 sp=0x170ac6923fb8 pc=0x22d999f\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6923fe8 sp=0x170ac6923fe0 pc=0x1f40801\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 109\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n\ngoroutine 115 gp=0x170ac61d54a0 m=nil [select]:\nruntime.gopark(0x170ac6925f08?, 0x6?, 0x38?, 0x5b?, 0x170ac6925da4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6925c08 sp=0x170ac6925be8 pc=0x1f38d2e\nruntime.selectgo(0x170ac6925f08, 0x170ac6925d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6925d38 sp=0x170ac6925c08 pc=0x1f152a5\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1680, 0x170ac699e500, 0x0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6925f90 sp=0x170ac6925d38 pc=0x22da6a8\nnet/http.(*http2clientStream).doRequest(0x170ac61a1680, 0x0?, 0x170ac66e1f98?)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6925fb8 sp=0x170ac6925f90 pc=0x22d99d8\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6925fe0 sp=0x170ac6925fb8 pc=0x22d999f\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6925fe8 sp=0x170ac6925fe0 pc=0x1f40801\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 107\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n\nrax    0x0\nrbx    0x7f7eabfff640\nrcx    0x7f7ef9baa94c\nrdx    0x6\nrdi    0x10a2e\nrsi    0x10a30\nrbp    0x10a30\nrsp    0x7f7eabffe790\nr8     0x7f7eabffe860\nr9     0x7f7ef9cd04e0\nr10    0x8\nr11    0x246\nr12    0x6\nr13    0x7f7eabffeb60\nr14    0x170ac5c1f0e0\nr15    0xffffffffffffffff\nrip    0x7f7ef9baa94c\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\n\n  query-infra-both: command failed: exit status 2\noutput: runtime/cgo: pthread_create failed: Resource temporarily unavailable\nSIGABRT: abort\nPC=0x7f5e9c16994c m=2 sigcode=18446744073709551610\n\ngoroutine 0 gp=0x280029a48b40 m=2 mp=0x280029abb008 [idle]:\nruntime: g 0 gp=0x280029a48b40: unknown pc 0x7f5e9c16994c\nstack: frame={sp:0x7f5e554a7740, fp:0x0} stack=[0x7f5e54ca81e0,0x7f5e554a7de0)\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n00007f5e554a7640: 00000000 01eb64e5  00000000 00000000  .d..............\n                  ^ <internal/runtime/syscall/linux.EpollWait+0x45>\n00007f5e554a7650: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7660: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7670: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7680: 00007f5e 554a76d4  00007f5e 554a7d18  .vJU^....}JU^...\n00007f5e554a7690: 00000000 01efa894  00000000 00000000  ................\n                  ^ <runtime.netpoll+0xd4>\n00007f5e554a76a0: 00007f5e 554a76d4  00000000 00000000  .vJU^...........\n00007f5e554a76b0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a76c0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a76d0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a76e0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a76f0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7700: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7710: 00000000 00000000  00007f5e 9c176fa9  .........o..^...\n00007f5e554a7720: 00000000 00000000  00000000 00000011  ................\n00007f5e554a7730: 00000000 00000130  00007f5e 9c16993e  0.......>...^...\n00007f5e554a7740: 00000000 00000120  00000000 00000000   ...............\n                  ^ SP\n00007f5e554a7750: 00000000 00000013  00000000 00000000  ................\n00007f5e554a7760: 00000000 00000000  00007f5e 9c1eccff  ............^...\n00007f5e554a7770: 00007f5e 327fc910  00007f5e 9c1eccb1  ...2^.......^...\n00007f5e554a7780: 00007f5e 327fc640  00007f5e 554a7a50  @..2^...PzJU^...\n00007f5e554a7790: 00007f5e 554a78be  00007f5e 554a78bf  .xJU^....xJU^...\n00007f5e554a77a0: 00007f5e 327fc640  00007f5e 9c1677f5  @..2^....w..^...\n00007f5e554a77b0: 00007f5e 50000020  00000000 00000120   ..P^... .......\n00007f5e554a77c0: 00000000 003d0f00  5a23f32c fe4c8400  ..=.......L.,.#Z\n00007f5e554a77d0: 00007f5e 554a8640  00000000 00000006  @.JU^...........\n00007f5e554a77e0: 00000000 0bac42a0  00007f5e 554a7b10  .B.......{JU^...\n00007f5e554a77f0: 00002800 29a48b40  00007f5e 9c11c646  @..).(..F...^...\n00007f5e554a7800: 00007f5e 9c2d9e90  00007f5e 9c1067f3  ..-.^....g..^...\n00007f5e554a7810: 00000000 00000020  5a23f32c fe4c8400   .........L.,.#Z\n00007f5e554a7820: 00007f5e 31ffc000  00007f5e 9c1dbac4  ...1^.......^...\n00007f5e554a7830: 00000000 00000000  00000000 0000000d  ................\nruntime: g 0 gp=0x280029a48b40: unknown pc 0x7f5e9c16994c\nstack: frame={sp:0x7f5e554a7740, fp:0x0} stack=[0x7f5e54ca81e0,0x7f5e554a7de0)\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n00007f5e554a7640: 00000000 01eb64e5  00000000 00000000  .d..............\n                  ^ <internal/runtime/syscall/linux.EpollWait+0x45>\n00007f5e554a7650: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7660: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7670: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7680: 00007f5e 554a76d4  00007f5e 554a7d18  .vJU^....}JU^...\n00007f5e554a7690: 00000000 01efa894  00000000 00000000  ................\n                  ^ <runtime.netpoll+0xd4>\n00007f5e554a76a0: 00007f5e 554a76d4  00000000 00000000  .vJU^...........\n00007f5e554a76b0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a76c0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a76d0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a76e0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a76f0: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7700: 00000000 00000000  00000000 00000000  ................\n00007f5e554a7710: 00000000 00000000  00007f5e 9c176fa9  .........o..^...\n00007f5e554a7720: 00000000 00000000  00000000 00000011  ................\n00007f5e554a7730: 00000000 00000130  00007f5e 9c16993e  0.......>...^...\n00007f5e554a7740: 00000000 00000120  00000000 00000000   ...............\n                  ^ SP\n00007f5e554a7750: 00000000 00000013  00000000 00000000  ................\n00007f5e554a7760: 00000000 00000000  00007f5e 9c1eccff  ............^...\n00007f5e554a7770: 00007f5e 327fc910  00007f5e 9c1eccb1  ...2^.......^...\n00007f5e554a7780: 00007f5e 327fc640  00007f5e 554a7a50  @..2^...PzJU^...\n00007f5e554a7790: 00007f5e 554a78be  00007f5e 554a78bf  .xJU^....xJU^...\n00007f5e554a77a0: 00007f5e 327fc640  00007f5e 9c1677f5  @..2^....w..^...\n00007f5e554a77b0: 00007f5e 50000020  00000000 00000120   ..P^... .......\n00007f5e554a77c0: 00000000 003d0f00  5a23f32c fe4c8400  ..=.......L.,.#Z\n00007f5e554a77d0: 00007f5e 554a8640  00000000 00000006  @.JU^...........\n00007f5e554a77e0: 00000000 0bac42a0  00007f5e 554a7b10  .B.......{JU^...\n00007f5e554a77f0: 00002800 29a48b40  00007f5e 9c11c646  @..).(..F...^...\n00007f5e554a7800: 00007f5e 9c2d9e90  00007f5e 9c1067f3  ..-.^....g..^...\n00007f5e554a7810: 00000000 00000020  5a23f32c fe4c8400   .........L.,.#Z\n00007f5e554a7820: 00007f5e 31ffc000  00007f5e 9c1dbac4  ...1^.......^...\n00007f5e554a7830: 00000000 00000000  00000000 0000000d  ................\n\ngoroutine 1 gp=0x280029a481e0 m=nil [sync.WaitGroup.Wait]:\nruntime.gopark(0x28002a5927b8?, 0x2022a4d?, 0x30?, 0xe9?, 0x28002a5927e8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a5927a0 sp=0x28002a592780 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x280029ae34d0, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a592808 sp=0x28002a5927a0 pc=0x1f162b2\nsync.runtime_SemacquireWaitGroup(0x28002a83b578?, 0xe0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x28002a592840 sp=0x28002a592808 pc=0x1f3a66e\nsync.(*WaitGroup).Wait(0x280029ae34c8)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x28002a592868 sp=0x28002a592840 pc=0x1f4f445\ngolang.org/x/sync/errgroup.(*Group).Wait(0x280029ae34c0)\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x28002a592888 sp=0x28002a592868 pc=0x362507e\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0}, {0x28002a977b50, 0x1, 0x1})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x28002a592948 sp=0x28002a592888 pc=0x3927e05\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).gatherInfraLogs(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:587 +0x867 fp=0x28002a592d30 sp=0x28002a592948 pc=0x3927727\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:333 +0x533 fp=0x28002a593908 sp=0x28002a592d30 pc=0x3921bf3\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedInfraQueryOptions).RunInfra(0x28002a11e1e0, {0x59f0b08, 0x28002a1873b0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:83 +0x71f fp=0x28002a593b30 sp=0x28002a593908 pc=0x393887f\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawInfraQueryOptions).Run(0x0?, {0x59f0b08, 0x28002a1873b0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_options.go:109 +0x54 fp=0x28002a593b58 sp=0x28002a593b30 pc=0x3939134\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryInfraCommand.func1(0x28002a53d508?, {0x280029ca3b00?, 0x4?, 0x553b0e2?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:45 +0x28 fp=0x28002a593b80 sp=0x28002a593b58 pc=0x3938128\ngithub.com/spf13/cobra.(*Command).execute(0x28002a53d508, {0x280029ab8120, 0xc, 0xc})\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x28002a593d28 sp=0x28002a593b80 pc=0x212e614\ngithub.com/spf13/cobra.(*Command).ExecuteC(0x280029ec9b08)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x28002a593e18 sp=0x28002a593d28 pc=0x212efa5\ngithub.com/spf13/cobra.(*Command).Execute(...)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071\ngithub.com/spf13/cobra.(*Command).ExecuteContext(...)\n\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064\nmain.main()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x28002a593f48 sp=0x28002a593e18 pc=0x47ddd6a\nruntime.main()\n\t/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x28002a593fe0 sp=0x28002a593f48 pc=0x1f01975\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a593fe8 sp=0x28002a593fe0 pc=0x1f40801\n\ngoroutine 2 gp=0x280029a48d20 m=nil [force gc (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b08fa8 sp=0x280029b08f88 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.forcegchelper()\n\t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x280029b08fe0 sp=0x280029b08fa8 pc=0x1f01c93\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b08fe8 sp=0x280029b08fe0 pc=0x1f40801\ncreated by runtime.init.7 in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\n\ngoroutine 3 gp=0x280029a492c0 m=nil [GC sweep wait]:\nruntime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b09788 sp=0x280029b09768 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.bgsweep(0x280029b30000)\n\t/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x280029b097c8 sp=0x280029b09788 pc=0x1ee9f11\nruntime.gcenable.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x280029b097e0 sp=0x280029b097c8 pc=0x1edadd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b097e8 sp=0x280029b097e0 pc=0x1f40801\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\n\ngoroutine 4 gp=0x280029a494a0 m=nil [GC scavenge wait]:\nruntime.gopark(0x10000?, 0x5981cd8?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b09f78 sp=0x280029b09f58 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.(*scavengerState).park(0x82979c0)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x280029b09fa8 sp=0x280029b09f78 pc=0x1ee77e9\nruntime.bgscavenge(0x280029b30000)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x280029b09fc8 sp=0x280029b09fa8 pc=0x1ee7d99\nruntime.gcenable.gowrap2()\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x280029b09fe0 sp=0x280029b09fc8 pc=0x1edad97\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b09fe8 sp=0x280029b09fe0 pc=0x1f40801\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\n\ngoroutine 5 gp=0x280029a49a40 m=nil [GOMAXPROCS updater (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0a788 sp=0x280029b0a768 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.updateMaxProcsGoroutine()\n\t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x280029b0a7e0 sp=0x280029b0a788 pc=0x1f10267\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0a7e8 sp=0x280029b0a7e0 pc=0x1f40801\ncreated by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\n\ngoroutine 6 gp=0x280029b481e0 m=nil [finalizer wait]:\nruntime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x280029b08601?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b08620 sp=0x280029b08600 pc=0x1f38d2e\nruntime.runFinalizers()\n\t/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x280029b087e0 sp=0x280029b08620 pc=0x1ed9cc7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b087e8 sp=0x280029b087e0 pc=0x1f40801\ncreated by runtime.createfing in goroutine 1\n\t/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d\n\ngoroutine 7 gp=0x280029d3fc20 m=nil [cleanup wait]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0af68 sp=0x280029b0af48 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.(*cleanupQueue).dequeue(0x82980e0)\n\t/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x280029b0afa0 sp=0x280029b0af68 pc=0x1ed6074\nruntime.runCleanups()\n\t/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x280029b0afe0 sp=0x280029b0afa0 pc=0x1ed66e5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0afe8 sp=0x280029b0afe0 pc=0x1f40801\ncreated by runtime.(*cleanupQueue).createGs in goroutine 1\n\t/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5\n\ngoroutine 8 gp=0x280029ee6000 m=nil [GC worker (idle)]:\nruntime.gopark(0x125c86ae32d7?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0b740 sp=0x280029b0b720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x280029ad7ab0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x280029b0b7c8 sp=0x280029b0b740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b0b7e0 sp=0x280029b0b7c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0b7e8 sp=0x280029b0b7e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 18 gp=0x280029b843c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x125c662e5759?, 0x3?, 0xc2?, 0x10?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b04740 sp=0x280029b04720 pc=0x1f38d2e\nruntime.gcBgMarkWorker(0x280029ad7ab0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x280029b047c8 sp=0x280029b04740 pc=0x1edd8ab\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b047e0 sp=0x280029b047c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b047e8 sp=0x280029b047e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 9 gp=0x280029ee61e0 m=nil [semacquire]:\nruntime.gopark(0x6e2b22?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029b0bc96?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0be38 sp=0x280029b0be18 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029b0bea0 sp=0x280029b0be38 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029b0bf40 sp=0x280029b0bea0 pc=0x1edbb12\nruntime.gcBgMarkWorker(0x280029ad7ab0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029b0bfc8 sp=0x280029b0bf40 pc=0x1edda88\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b0bfe0 sp=0x280029b0bfc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0bfe8 sp=0x280029b0bfe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 10 gp=0x280029ee63c0 m=nil [semacquire]:\nruntime.gopark(0x1ec6f5e?, 0x1f3ec12?, 0x80?, 0x51?, 0x1ec70df?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eec638 sp=0x280029eec618 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eec6a0 sp=0x280029eec638 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eec740 sp=0x280029eec6a0 pc=0x1edbb12\nruntime.gcBgMarkWorker(0x280029ad7ab0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eec7c8 sp=0x280029eec740 pc=0x1edda88\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eec7e0 sp=0x280029eec7c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eec7e8 sp=0x280029eec7e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 11 gp=0x280029ee65a0 m=nil [semacquire]:\nruntime.gopark(0x54b522?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029eecc96?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eece38 sp=0x280029eece18 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eecea0 sp=0x280029eece38 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eecf40 sp=0x280029eecea0 pc=0x1edbb12\nruntime.gcBgMarkWorker(0x280029ad7ab0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eecfc8 sp=0x280029eecf40 pc=0x1edda88\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eecfe0 sp=0x280029eecfc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eecfe8 sp=0x280029eecfe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 34 gp=0x280029f04000 m=nil [semacquire]:\nruntime.gopark(0x2d7122?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029ee8496?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029ee8638 sp=0x280029ee8618 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029ee86a0 sp=0x280029ee8638 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029ee8740 sp=0x280029ee86a0 pc=0x1edbb12\nruntime.gcBgMarkWorker(0x280029ad7ab0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029ee87c8 sp=0x280029ee8740 pc=0x1edda88\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029ee87e0 sp=0x280029ee87c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029ee87e8 sp=0x280029ee87e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 19 gp=0x280029b845a0 m=4 mp=0x280029b34008 [GC mark termination]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x280029b04ea0 sp=0x280029b04e90 pc=0x1f3ebc8\nruntime.forEachP(...)\n\t/usr/lib/golang/src/runtime/proc.go:2112\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x280029b04f40 sp=0x280029b04ea0 pc=0x1edbc34\nruntime.gcBgMarkWorker(0x280029ad7ab0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029b04fc8 sp=0x280029b04f40 pc=0x1edda88\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b04fe0 sp=0x280029b04fc8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b04fe8 sp=0x280029b04fe0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 12 gp=0x280029ee6780 m=nil [semacquire]:\nruntime.gopark(0x3f9e22?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029eed496?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eed638 sp=0x280029eed618 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eed6a0 sp=0x280029eed638 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eed740 sp=0x280029eed6a0 pc=0x1edbb12\nruntime.gcBgMarkWorker(0x280029ad7ab0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eed7c8 sp=0x280029eed740 pc=0x1edda88\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eed7e0 sp=0x280029eed7c8 pc=0x1edd797\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eed7e8 sp=0x280029eed7e0 pc=0x1f40801\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 35 gp=0x280029ee74a0 m=nil [select, locked to thread]:\nruntime.gopark(0x280029eef7a8?, 0x2?, 0x1?, 0x74?, 0x280029eef794?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eef628 sp=0x280029eef608 pc=0x1f38d2e\nruntime.selectgo(0x280029eef7a8, 0x280029eef790, 0x0?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x280029eef758 sp=0x280029eef628 pc=0x1f152a5\nruntime.ensureSigM.func1()\n\t/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x280029eef7e0 sp=0x280029eef758 pc=0x1f328e8\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eef7e8 sp=0x280029eef7e0 pc=0x1f40801\ncreated by runtime.ensureSigM in goroutine 1\n\t/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5\n\ngoroutine 50 gp=0x280029f84960 m=9 mp=0x28002a000008 [syscall]:\nruntime.notetsleepg(0x82bdea0, 0xffffffffffffffff)\n\t/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x280029ee9fa0 sp=0x280029ee9f78 pc=0x1ece509\nos/signal.signal_recv()\n\t/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x280029ee9fc0 sp=0x280029ee9fa0 pc=0x1f3b298\nos/signal.loop()\n\t/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x280029ee9fe0 sp=0x280029ee9fc0 pc=0x2030553\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029ee9fe8 sp=0x280029ee9fe0 pc=0x1f40801\ncreated by os/signal.Notify.func1.1 in goroutine 1\n\t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\n\ngoroutine 51 gp=0x280029f84b40 m=nil [select]:\nruntime.gopark(0x280029eea798?, 0x2?, 0x80?, 0xa3?, 0x280029eea78c?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eea628 sp=0x280029eea608 pc=0x1f38d2e\nruntime.selectgo(0x280029eea798, 0x280029eea788, 0x0?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x280029eea758 sp=0x280029eea628 pc=0x1f152a5\nos/signal.NotifyContext.func1()\n\t/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x280029eea7e0 sp=0x280029eea758 pc=0x202ff9c\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eea7e8 sp=0x280029eea7e0 pc=0x1f40801\ncreated by os/signal.NotifyContext in goroutine 1\n\t/usr/lib/golang/src/os/signal/signal.go:291 +0x158\n\ngoroutine 137 gp=0x28002a60a000 m=nil [sync.WaitGroup.Wait]:\nruntime.gopark(0x28002a603e00?, 0x1ed0765?, 0x80?, 0xea?, 0x7f5e54234a40?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a603da0 sp=0x28002a603d80 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x280029ae3b10, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a603e08 sp=0x28002a603da0 pc=0x1f162b2\nsync.runtime_SemacquireWaitGroup(0x28002a83b590?, 0x0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x28002a603e40 sp=0x28002a603e08 pc=0x1f3a66e\nsync.(*WaitGroup).Wait(0x280029ae3b08)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x28002a603e68 sp=0x28002a603e40 pc=0x1f4f445\ngolang.org/x/sync/errgroup.(*Group).Wait(0x280029ae3b00)\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x28002a603e88 sp=0x28002a603e68 pc=0x362507e\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x280029d80e80, {0x59f0b40, 0x280029beca00}, {0x28002a977b50, 0x1, 0x38d788c?}, 0x28002a830c40)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x28002a603f00 sp=0x28002a603e88 pc=0x392976a\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x28002a603f78 sp=0x28002a603f00 pc=0x392808e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a603fe0 sp=0x28002a603f78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a603fe8 sp=0x28002a603fe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 38 gp=0x28002a60b0e0 m=nil [IO wait]:\nruntime.gopark(0x28002a7c22c0?, 0x280029aee7f8?, 0x40?, 0x77?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b197b8 sp=0x280029b19798 pc=0x1f38d2e\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x280029b197f0 sp=0x280029b197b8 pc=0x1ef9e57\ninternal/poll.runtime_pollWait(0x7f5e554b6c00, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x280029b19810 sp=0x280029b197f0 pc=0x1f375c5\ninternal/poll.(*pollDesc).wait(0x280029bf6400?, 0x28002a90c000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x280029b19838 sp=0x280029b19810 pc=0x1fbd167\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0x280029bf6400, {0x28002a90c000, 0x3500, 0x3500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x280029b198d0 sp=0x280029b19838 pc=0x1fbe4ce\nnet.(*netFD).Read(0x280029bf6400, {0x28002a90c000?, 0x28002a90c000?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x280029b19918 sp=0x280029b198d0 pc=0x20798c5\nnet.(*conn).Read(0x280029a46378, {0x28002a90c000?, 0x7f5e554a9828?, 0x7f5e9c0b1e00?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x280029b19960 sp=0x280029b19918 pc=0x208b7a5\ncrypto/tls.(*atLeastReader).Read(0x28002aa26ba0, {0x28002a90c000?, 0x28002a0d8658?, 0x1efcc28?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x280029b199a8 sp=0x280029b19960 pc=0x2222ddb\nbytes.(*Buffer).ReadFrom(0x28002a0d8628, {0x59a3680, 0x28002aa26ba0})\n\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x280029b19a00 sp=0x280029b199a8 pc=0x1fe9258\ncrypto/tls.(*Conn).readFromUntil(0x28002a0d8388, {0x59a32c0, 0x280029a46378}, 0x222764c?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x280029b19a38 sp=0x280029b19a00 pc=0x2222fbe\ncrypto/tls.(*Conn).readRecordOrCCS(0x28002a0d8388, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x280029b19c18 sp=0x280029b19a38 pc=0x222009b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:588\ncrypto/tls.(*Conn).Read(0x28002a0d8388, {0x280029fed000, 0x1000, 0x22c34e0?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x280029b19c80 sp=0x280029b19c18 pc=0x22267c5\nbufio.(*Reader).Read(0x28002a7b89c0, {0x28002a7c8584, 0x9, 0x22e100e?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0x280029b19cb8 sp=0x280029b19c80 pc=0x20a71f7\nio.ReadAtLeast({0x59a1260, 0x28002a7b89c0}, {0x28002a7c8584, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0x280029b19d00 sp=0x280029b19cb8 pc=0x1fb17ee\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\nnet/http.http2readFrameHeader({0x28002a7c8584, 0x9, 0x280000000015?}, {0x59a1260?, 0x28002a7b89c0?})\n\t/usr/lib/golang/src/net/http/h2_bundle.go:1805 +0x65 fp=0x280029b19d50 sp=0x280029b19d00 pc=0x22b9e25\nnet/http.(*http2Framer).ReadFrameHeader(0x28002a7c8540)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:2071 +0x6b fp=0x280029b19dd8 sp=0x280029b19d50 pc=0x22ba9eb\nnet/http.(*http2Framer).ReadFrame(0x28002a7c8540)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:2130 +0x18 fp=0x280029b19e18 sp=0x280029b19dd8 pc=0x22bb1b8\nnet/http.(*http2clientConnReadLoop).run(0x280029b19fa8)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9550 +0xca fp=0x280029b19f60 sp=0x280029b19e18 pc=0x22de14a\nnet/http.(*http2ClientConn).readLoop(0x280029bc01c0)\n\t/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x280029b19fc8 sp=0x280029b19f60 pc=0x22dd532\nnet/http.(*http2Transport).newClientConn.gowrap1()\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x280029b19fe0 sp=0x280029b19fc8 pc=0x22d7057\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b19fe8 sp=0x280029b19fe0 pc=0x1f40801\ncreated by net/http.(*http2Transport).newClientConn in goroutine 37\n\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5\n\ngoroutine 69 gp=0x28002a60b4a0 m=nil [IO wait]:\nruntime.gopark(0x28002a50c560?, 0x1edb176?, 0x80?, 0xed?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a606808 sp=0x28002a6067e8 pc=0x1f38d2e\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x28002a606840 sp=0x28002a606808 pc=0x1ef9e57\ninternal/poll.runtime_pollWait(0x7f5e554b6a00, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x28002a606860 sp=0x28002a606840 pc=0x1f375c5\ninternal/poll.(*pollDesc).wait(0x280029bf6c80?, 0x280029a5a000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x28002a606888 sp=0x28002a606860 pc=0x1fbd167\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0x280029bf6c80, {0x280029a5a000, 0x2500, 0x2500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x28002a606920 sp=0x28002a606888 pc=0x1fbe4ce\nnet.(*netFD).Read(0x280029bf6c80, {0x280029a5a000?, 0x28002a60690a?, 0x1edb176?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x28002a606968 sp=0x28002a606920 pc=0x20798c5\nnet.(*conn).Read(0x280029a463c0, {0x280029a5a000?, 0x7f5e5423eae0?, 0x7f5e9c09dde0?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x28002a6069b0 sp=0x28002a606968 pc=0x208b7a5\ncrypto/tls.(*atLeastReader).Read(0x28002a628318, {0x280029a5a000?, 0x28002a606a98?, 0x1efcc28?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x28002a6069f8 sp=0x28002a6069b0 pc=0x2222ddb\nbytes.(*Buffer).ReadFrom(0x280029ba9ea8, {0x59a3680, 0x28002a628318})\n\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x28002a606a50 sp=0x28002a6069f8 pc=0x1fe9258\ncrypto/tls.(*Conn).readFromUntil(0x280029ba9c08, {0x59a32c0, 0x280029a463c0}, 0x222764c?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x28002a606a88 sp=0x28002a606a50 pc=0x2222fbe\ncrypto/tls.(*Conn).readRecordOrCCS(0x280029ba9c08, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x28002a606c68 sp=0x28002a606a88 pc=0x222009b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:588\ncrypto/tls.(*Conn).Read(0x280029ba9c08, {0x28002a24a000, 0x1000, 0x4e55540?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x28002a606cd0 sp=0x28002a606c68 pc=0x22267c5\nnet/http.(*persistConn).Read(0x28002a0ad400, {0x28002a24a000?, 0x59a0ea0?, 0x81260e0?})\n\t/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x28002a606d30 sp=0x28002a606cd0 pc=0x2319d07\nbufio.(*Reader).fill(0x28002a8130e0)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x28002a606d68 sp=0x28002a606d30 pc=0x20a6c63\nbufio.(*Reader).Peek(0x28002a8130e0, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x28002a606d80 sp=0x28002a606d68 pc=0x20a6df3\nnet/http.(*persistConn).readLoop(0x28002a0ad400)\n\t/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x28002a606fc8 sp=0x28002a606d80 pc=0x231a812\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x28002a606fe0 sp=0x28002a606fc8 pc=0x2319197\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a606fe8 sp=0x28002a606fe0 pc=0x1f40801\ncreated by net/http.(*Transport).dialConn in goroutine 55\n\t/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac\n\ngoroutine 70 gp=0x28002a60bc20 m=nil [select]:\nruntime.gopark(0x28002a613f48?, 0x2?, 0x0?, 0xb6?, 0x28002a613ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a708d88 sp=0x28002a708d68 pc=0x1f38d2e\nruntime.selectgo(0x28002a708f48, 0x28002a613ef0, 0x280029c93a80?, 0x0, 0x28002a4f7c80?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a708eb8 sp=0x28002a708d88 pc=0x1f152a5\nnet/http.(*persistConn).writeLoop(0x28002a0ad400)\n\t/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x28002a708fc8 sp=0x28002a708eb8 pc=0x231c426\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x28002a708fe0 sp=0x28002a708fc8 pc=0x2319157\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a708fe8 sp=0x28002a708fe0 pc=0x1f40801\ncreated by net/http.(*Transport).dialConn in goroutine 55\n\t/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe\n\ngoroutine 138 gp=0x28002a84a780 m=nil [semacquire]:\nruntime.gopark(0x1?, 0x1f78fa5?, 0xe0?, 0x75?, 0x2?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a2ca910 sp=0x28002a2ca8f0 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a2ca978 sp=0x28002a2ca910 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x28002a2caa18 sp=0x28002a2ca978 pc=0x1edbb12\nruntime.gcAssistAlloc(0x28002a84a780)\n\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x28002a2caaa8 sp=0x28002a2caa18 pc=0x1ee06b6\nruntime.deductAssistCredit(0x28002a2cac70?)\n\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x28002a2caac8 sp=0x28002a2caaa8 pc=0x1ed10a6\nruntime.mallocgc(0x10, 0x4af39e0, 0x1)\n\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x28002a2caaf8 sp=0x28002a2caac8 pc=0x1f35e07\nruntime.convTstring({0x28002a6a7728, 0x14})\n\t/usr/lib/golang/src/runtime/iface.go:431 +0x45 fp=0x28002a2cab20 sp=0x28002a2caaf8 pc=0x1f35865\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x280029beca00}, 0x28002a830cb0, 0x280029f636e0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:297 +0x394 fp=0x28002a2caf48 sp=0x28002a2cab20 pc=0x3920674\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x28002a2caf78 sp=0x28002a2caf48 pc=0x392800c\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a2cafe0 sp=0x28002a2caf78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a2cafe8 sp=0x28002a2cafe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 139 gp=0x28002a84a960 m=nil [select]:\nruntime.gopark(0x28002a4bdd00?, 0x2?, 0x1?, 0xc9?, 0x28002a4bdc74?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a4bdac8 sp=0x28002a4bdaa8 pc=0x1f38d2e\nruntime.selectgo(0x28002a4bdd00, 0x28002a4bdc70, 0x28002a3e8a50?, 0x0, 0x280029f827c8?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a4bdbf8 sp=0x28002a4bdac8 pc=0x1f152a5\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x49424d554554534c?, {0x59f0b40, 0x280029beca00}, 0x28002a830c40, 0x28002a830cb0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x28002a4bdf10 sp=0x28002a4bdbf8 pc=0x392836e\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x28002a4bdf78 sp=0x28002a4bdf10 pc=0x3927f3d\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a4bdfe0 sp=0x28002a4bdf78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a4bdfe8 sp=0x28002a4bdfe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 140 gp=0x28002a84ab40 m=nil [semacquire]:\nruntime.gopark(0x1f6aa32?, 0x28002a34b418?, 0xc5?, 0xbd?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a34b3c0 sp=0x28002a34b3a0 pc=0x1f38d2e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a34b428 sp=0x28002a34b3c0 pc=0x1f162b2\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x28002a34b4c8 sp=0x28002a34b428 pc=0x1edbb12\nruntime.gcAssistAlloc(0x28002a84ab40)\n\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x28002a34b558 sp=0x28002a34b4c8 pc=0x1ee06b6\nruntime.deductAssistCredit(0x20?)\n\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x28002a34b578 sp=0x28002a34b558 pc=0x1ed10a6\nruntime.mallocgc(0x1000, 0x4af32a0, 0x1)\n\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x28002a34b5a8 sp=0x28002a34b578 pc=0x1f35e07\nruntime.makeslice(0x28002a34b668?, 0x1f8?, 0x1ff?)\n\t/usr/lib/golang/src/runtime/slice.go:117 +0x49 fp=0x28002a34b5d0 sp=0x28002a34b5a8 pc=0x1f3b489\nbufio.NewWriterSize(...)\n\t/usr/lib/golang/src/bufio/bufio.go:599\nbufio.NewWriter(...)\n\t/usr/lib/golang/src/bufio/bufio.go:608\nencoding/csv.NewWriter(...)\n\t/usr/lib/golang/src/encoding/csv/writer.go:42\ngithub.com/Azure/azure-kusto-go/azkustodata/query.(*row).String(0x812ab90?)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/row_impl.go:100 +0x18e fp=0x28002a34b6e8 sp=0x28002a34b5d0 pc=0x38cf5ee\nfmt.(*pp).handleMethods(0x28002a24c0d0, 0x2a34b8f0?)\n\t/usr/lib/golang/src/fmt/print.go:674 +0x2f4 fp=0x28002a34b8a8 sp=0x28002a34b6e8 pc=0x1fdd1d4\nfmt.(*pp).printArg(0x28002a24c0d0, {0x549a3a0, 0x28002a7c7c80}, 0x76)\n\t/usr/lib/golang/src/fmt/print.go:757 +0x42e fp=0x28002a34b938 sp=0x28002a34b8a8 pc=0x1fdd9ee\nfmt.(*pp).doPrintf(0x28002a24c0d0, {0x553a400, 0x2}, {0x28002a34bd08, 0x1, 0x1})\n\t/usr/lib/golang/src/fmt/print.go:1075 +0x3ec fp=0x28002a34ba20 sp=0x28002a34b938 pc=0x1fe04ec\nfmt.Sprintf({0x553a400, 0x2}, {0x28002a34bd08, 0x1, 0x1})\n\t/usr/lib/golang/src/fmt/print.go:239 +0x53 fp=0x28002a34ba78 sp=0x28002a34ba20 pc=0x1fda6b3\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x280029e454e0, {0x59f0b40, 0x280029beca50}, {0x5a00300, 0x280029ae2040}, 0x28002a830c40)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:160 +0x8d8 fp=0x28002a34bd60 sp=0x28002a34ba78 pc=0x391bad8\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x2d20414320746f6f?, {0x59f0b40?, 0x280029beca50?}, {0x5a00300?, 0x280029ae2040?}, 0x43437a4b4349494d?)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x28002a34be88 sp=0x28002a34bd60 pc=0x3929c16\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x28002a34bf78 sp=0x28002a34be88 pc=0x392984a\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a34bfe0 sp=0x28002a34bf78 pc=0x36251d0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a34bfe8 sp=0x28002a34bfe0 pc=0x1f40801\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 137\n\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 142 gp=0x28002a84af00 m=nil [select]:\nruntime.gopark(0x28002a333de0?, 0x2?, 0x8?, 0xb8?, 0x28002a333dbc?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a333c58 sp=0x28002a333c38 pc=0x1f38d2e\nruntime.selectgo(0x28002a333de0, 0x28002a333db8, 0x280029afc008?, 0x1, 0x1f3665f?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a333d88 sp=0x28002a333c58 pc=0x1f152a5\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.(*iterativeDataset).sendTable(0x280029becf00, {0x5a19be0, 0x28002ab876c0})\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:378 +0xf3 fp=0x28002a333e10 sp=0x28002a333d88 pc=0x38d8d33\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.handleDataTable(0x280029becf00, 0x28002a3b2b40)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:324 +0x2c6 fp=0x28002a333ec8 sp=0x28002a333e10 pc=0x38d8686\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x280029becf00)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:167 +0x148 fp=0x28002a333f30 sp=0x28002a333ec8 pc=0x38d7b48\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x280029becf00, 0x28002a977da0)\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x28002a333fc0 sp=0x28002a333f30 pc=0x38d788c\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x28002a333fe0 sp=0x28002a333fc0 pc=0x38d751b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a333fe8 sp=0x28002a333fe0 pc=0x1f40801\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 140\n\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc\n\nrax    0x0\nrbx    0x7f5e554a8640\nrcx    0x7f5e9c16994c\nrdx    0x6\nrdi    0x10a66\nrsi    0x10a67\nrbp    0x10a67\nrsp    0x7f5e554a7740\nr8     0x7f5e554a7810\nr9     0x7f5e9c28f4e0\nr10    0x8\nr11    0x246\nr12    0x6\nr13    0x7f5e554a7b10\nr14    0x280029a48b40\nr15    0xffffffffffffffff\nrip    0x7f5e9c16994c\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\n",
    }

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T21:37:00Z
  Ended:       2026-10-15T21:43:02Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2049792-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfev967s3oovrdjqq5mq77ir2sdcesj-h1m5k2y2a1v0t2s", pod="kube-apiserver-787b5d5c6d-g8v2j", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfev967s3oovrdjqq5mq77ir2sdcesj-h1m5k2y2a1v0t2s/kube-apiserver-787b5d5c6d-g8v2j has been in a non-ready state for longer than 5 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.006s.
Expected success, but got an error:
    <*fmt.wrapError | 0x39916b3f7320>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);
    let _endTime = datetime(2026-10-15T21:07:31.8528726Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-54zkzjslmt86'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-15T21:07:45.2821972Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b", ActivityId:"203611ca-e380-4c98-a4a7-665b077207f6", SubActivityId:"203611ca-e380-4c98-a4a7-665b077207f6", ActivityType:"GW.Http.CallContext", ParentActivityId:"203611ca-e380-4c98-a4a7-665b077207f6", ActivityStack:"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 > GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:07:45.2821972Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\", ActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", SubActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityStack:\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 > GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x39916c0fc5a0>{
            errs: [
                <*fmt.wrapError | 0x39916b3f7300>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:07:45.2821972Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\", ActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", SubActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityStack:\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 > GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x39916b3f72e0>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:07:45.2821972Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\", ActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", SubActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityStack:\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 > GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x39916b897900>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:07:45.2821972Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\", ActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", SubActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityStack:\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 > GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x39916b8978c0>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:07:45.2821972Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\", ActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", SubActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityStack:\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 > GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x3991698bcf00>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.006s.
Expected success, but got an error:
    <*fmt.wrapError | 0x39916b3f7320>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);
    let _endTime = datetime(2026-10-15T21:07:31.8528726Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-54zkzjslmt86'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-15T21:07:45.2821972Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b", ActivityId:"203611ca-e380-4c98-a4a7-665b077207f6", SubActivityId:"203611ca-e380-4c98-a4a7-665b077207f6", ActivityType:"GW.Http.CallContext", ParentActivityId:"203611ca-e380-4c98-a4a7-665b077207f6", ActivityStack:"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 > GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:07:45.2821972Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\", ActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", SubActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityStack:\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 > GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x39916c0fc5a0>{
            errs: [
                <*fmt.wrapError | 0x39916b3f7300>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:07:45.2821972Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\", ActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", SubActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityStack:\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 > GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x39916b3f72e0>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:07:45.2821972Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\", ActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", SubActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityStack:\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 > GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x39916b897900>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:07:45.2821972Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\", ActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", SubActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityStack:\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 > GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x39916b8978c0>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:07:45.2821972Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\", ActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", SubActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityStack:\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 > GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x3991698bcf00>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }

alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T21:26:40Z
  Severity:    Sev3
  Labels:      alertname="LeaderElectionLeaseStale", cluster="ci01-j4315776-mgmt-2", component="leader-election", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.179:8080", job="kube-state-metrics", lease="work-manager", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4315776/providers/microsoft.monitor/accounts/services-j4315776", namespace="klusterlet-2tfeeaom86b60erngar03ad51ehpgqtm", pod="arohcp-monitor-kube-state-metrics-7988dc648c-h4l8s", prometheus="prometheus/prometheus", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Leader election lease work-manager in namespace klusterlet-2tfeeaom86b60erngar03ad51ehpgqtm on cluster ci01-j4315776-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk
Unexpected error:
    <*fmt.wrapError | 0x2a4934179d80>: 
    failed to create NodePool ephemeral-np: failed waiting for nodepool="ephemeral-np" for cluster "ephemeral-disk" in resourcegroup="ephemeral-osdisk-gtft8mgc5kwc" to finish creating: GET https://rp.j5701632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37156461-5409-48c2-85c8-26e90a95c4af
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37156461-5409-48c2-85c8-26e90a95c4af",
      "name": "37156461-5409-48c2-85c8-26e90a95c4af",
      "status": "Failed",
      "startTime": "2026-10-15T20:27:24.635428507Z",
      "endTime": "2026-10-15T20:46:31.281302981Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine ephemeral-disk-ephemeral-np-ghcfv-62gf2: InspectionFailed: Waiting for AzureMachine to report spec.providerID\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine ephemeral-disk-ephemeral-np-ghcfv-n2f47: Failed: availabilitysets failed to create or update. err: failed to get existing resource ephemeral-osdisk-gtft8mgc5kwc--managed/2tfeeaoo9r717nalfklm07mkpjq3ca2e_ephemeral-disk-ephemeral-np-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-gtft8mgc5kwc--managed/providers/Microsoft.Compute/availabilitySets/2tfeeaoo9r717nalfklm07mkpjq3ca2e_ephemeral-disk-ephemeral-np-as\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '8d1ff6d0-aadd-4331-9171-9443ac5f4337' exceeded. Please ... message truncated\nMachine ephemeral-disk-ephemeral-np-ghcfv-62gf2: NotReady: AzureMachine status.initialization.provisioned is false\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5701632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37156461-5409-48c2-85c8-26e90a95c4af
            correlation ID: 44913fa3-dfaa-4300-8f6a-77ee9b21e91f
            client request ID: b52ff7de-d76a-4d6c-8d81-14684e59505a
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk
Unexpected error:
    <*fmt.wrapError | 0x2a4934179d80>: 
    failed to create NodePool ephemeral-np: failed waiting for nodepool="ephemeral-np" for cluster "ephemeral-disk" in resourcegroup="ephemeral-osdisk-gtft8mgc5kwc" to finish creating: GET https://rp.j5701632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37156461-5409-48c2-85c8-26e90a95c4af
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37156461-5409-48c2-85c8-26e90a95c4af",
      "name": "37156461-5409-48c2-85c8-26e90a95c4af",
      "status": "Failed",
      "startTime": "2026-10-15T20:27:24.635428507Z",
      "endTime": "2026-10-15T20:46:31.281302981Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine ephemeral-disk-ephemeral-np-ghcfv-62gf2: InspectionFailed: Waiting for AzureMachine to report spec.providerID\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine ephemeral-disk-ephemeral-np-ghcfv-n2f47: Failed: availabilitysets failed to create or update. err: failed to get existing resource ephemeral-osdisk-gtft8mgc5kwc--managed/2tfeeaoo9r717nalfklm07mkpjq3ca2e_ephemeral-disk-ephemeral-np-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-gtft8mgc5kwc--managed/providers/Microsoft.Compute/availabilitySets/2tfeeaoo9r717nalfklm07mkpjq3ca2e_ephemeral-disk-ephemeral-np-as\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '8d1ff6d0-aadd-4331-9171-9443ac5f4337' exceeded. Please ... message truncated\nMachine ephemeral-disk-ephemeral-np-ghcfv-62gf2: NotReady: AzureMachine status.initialization.provisioned is false\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5701632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37156461-5409-48c2-85c8-26e90a95c4af
            correlation ID: 44913fa3-dfaa-4300-8f6a-77ee9b21e91f
            client request ID: b52ff7de-d76a-4d6c-8d81-14684e59505a
occurred

alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T20:53:22Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentRolloutStuck", cluster="ci01-j5701632-mgmt-1", component="kubernetes-infrastructure", condition="Progressing", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.167:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5701632/providers/microsoft.monitor/accounts/services-j5701632", namespace="open-cluster-management-agent-addon", pod="arohcp-monitor-kube-state-metrics-7988dc648c-mxqgp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="progressdeadlineexceeded", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", status="false"
  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-15T21:19:26Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentRolloutStuck", cluster="ci01-j5701632-mgmt-1", component="kubernetes-infrastructure", condition="Progressing", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.167:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5701632/providers/microsoft.monitor/accounts/services-j5701632", namespace="open-cluster-management-agent-addon", pod="arohcp-monitor-kube-state-metrics-7988dc648c-mxqgp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="progressdeadlineexceeded", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", status="false"
  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.

alert fired 3 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T20:43:22Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j5701632-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.167:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5701632/providers/microsoft.monitor/accounts/services-j5701632", namespace="open-cluster-management-agent-addon", pod="arohcp-monitor-kube-state-metrics-7988dc648c-mxqgp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-15T20:43:22Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j5701632-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.167:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5701632/providers/microsoft.monitor/accounts/services-j5701632", namespace="open-cluster-management-agent-addon", pod="arohcp-monitor-kube-state-metrics-7988dc648c-mxqgp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-15T20:52:22Z
  Ended:       2026-10-15T20:58:23Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j5701632-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.167:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5701632/providers/microsoft.monitor/accounts/services-j5701632", namespace="klusterlet-2tfee727vhjhqsf7g4e565boos8lkt5s", pod="arohcp-monitor-kube-state-metrics-7988dc648c-mxqgp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tfee727vhjhqsf7g4e565boos8lkt5s/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.

alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T20:18:21Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5701632-mgmt-1", component="kubernetes-infrastructure", namespace="open-cluster-management-agent-addon", pod="klusterlet-addon-workmgr-c4d6b786-v5xlx", region="centralus", severity="warning"
  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-c4d6b786-v5xlx has been in a non-ready state for longer than 5 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h2tfvwcdgj2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1db157ce8d00>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h2tfvwcdgj2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h2tfvwcdgj2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T20:17:33.3416665Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h2tfvwcdgj2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h2tfvwcdgj2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T20:17:33.3416665Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1db157ce8ce0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h2tfvwcdgj2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T20:17:33.3416665Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1db156deb090>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1db157ce8cc0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h2tfvwcdgj2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T20:17:33.3416665Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1db1586b9a70>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h2tfvwcdgj2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1db157ce8d00>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h2tfvwcdgj2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h2tfvwcdgj2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T20:17:33.3416665Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h2tfvwcdgj2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h2tfvwcdgj2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T20:17:33.3416665Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1db157ce8ce0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h2tfvwcdgj2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T20:17:33.3416665Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1db156deb090>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1db157ce8cc0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h2tfvwcdgj2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T20:17:33.3416665Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1db1586b9a70>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster "cluster-candidate-4-22-5bxn7w"
Unexpected error:
    <*fmt.wrapError | 0x37da2d0209a0>: 
    DNS for route host agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x37da2d0168c0>{
            msg: "DNS for agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster "cluster-candidate-4-22-5bxn7w"
Unexpected error:
    <*fmt.wrapError | 0x37da2d0209a0>: 
    DNS for route host agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x37da2d0168c0>{
            msg: "DNS for agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred

alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T21:16:42Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2571392-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.203:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2571392/providers/microsoft.monitor/accounts/hcps-j2571392", namespace="ocm-arohcpci01-2tfehdfsiuk7rspp81ggshks0ldi46jk-d0c0o9z7l8b4h1m", pod="arohcp-monitor-kube-state-metrics-7988dc648c-mdkdv", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tfehdfsiuk7rspp81ggshks0ldi46jk-d0c0o9z7l8b4h1m/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-15T21:16:42Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2571392-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.203:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2571392/providers/microsoft.monitor/accounts/hcps-j2571392", namespace="ocm-arohcpci01-2tfehdfsiuk7rspp81ggshks0ldi46jk-d0c0o9z7l8b4h1m", pod="arohcp-monitor-kube-state-metrics-7988dc648c-mdkdv", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tfehdfsiuk7rspp81ggshks0ldi46jk-d0c0o9z7l8b4h1m/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.

alert fired 7 time(s) (1 unknown, 6 known)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T20:35:37Z
  Ended:       2026-10-15T20:56:36Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2571392-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfeejbgccjroqp4igd8dfh2ubvsehma-l9u8p7u6d3w8o8k", pod="ingress-operator-7f68fbc968-7l8pt", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfeejbgccjroqp4igd8dfh2ubvsehma-l9u8p7u6d3w8o8k/ingress-operator-7f68fbc968-7l8pt has been in a non-ready state for longer than 5 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.
Expected success, but got an error:
    <*fmt.wrapError | 0x698a7204a40>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);
    let _endTime = datetime(2026-10-15T20:56:57.7600018Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-strwsplkk89b'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-15T20:57:10.6107763Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d", ActivityId:"0b2f5fe2-6a96-4579-a835-e7b034bb165a", SubActivityId:"0b2f5fe2-6a96-4579-a835-e7b034bb165a", ActivityType:"GW.Http.CallContext", ParentActivityId:"0b2f5fe2-6a96-4579-a835-e7b034bb165a", ActivityStack:"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a > GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-strwsplkk89b'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T20:57:10.6107763Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\", ActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", SubActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a > GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x698a61c09c0>{
            errs: [
                <*fmt.wrapError | 0x698a7204a20>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-strwsplkk89b'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T20:57:10.6107763Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\", ActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", SubActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a > GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x698a7204a00>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-strwsplkk89b'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T20:57:10.6107763Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\", ActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", SubActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a > GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x698a6aac4a0>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-strwsplkk89b'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T20:57:10.6107763Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\", ActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", SubActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a > GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x698a6aac460>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-strwsplkk89b'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T20:57:10.6107763Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\", ActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", SubActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a > GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x698a632ba00>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.
Expected success, but got an error:
    <*fmt.wrapError | 0x698a7204a40>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);
    let _endTime = datetime(2026-10-15T20:56:57.7600018Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-strwsplkk89b'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-15T20:57:10.6107763Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d", ActivityId:"0b2f5fe2-6a96-4579-a835-e7b034bb165a", SubActivityId:"0b2f5fe2-6a96-4579-a835-e7b034bb165a", ActivityType:"GW.Http.CallContext", ParentActivityId:"0b2f5fe2-6a96-4579-a835-e7b034bb165a", ActivityStack:"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a > GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-strwsplkk89b'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T20:57:10.6107763Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\", ActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", SubActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a > GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x698a61c09c0>{
            errs: [
                <*fmt.wrapError | 0x698a7204a20>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-strwsplkk89b'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T20:57:10.6107763Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\", ActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", SubActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a > GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x698a7204a00>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-strwsplkk89b'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T20:57:10.6107763Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\", ActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", SubActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a > GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x698a6aac4a0>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-strwsplkk89b'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T20:57:10.6107763Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\", ActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", SubActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a > GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x698a6aac460>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-strwsplkk89b'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T20:57:10.6107763Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\", ActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", SubActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a > GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x698a632ba00>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }

time=2026-10-15T18:41:38.937Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description="Step cluster\n  Kind: ARM\n  Template: templates/svc-cluster.bicep\n  Parameters: configurations/svc-cluster.tmpl.bicepparam"
time=2026-10-15T18:41:40.440Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster
time=2026-10-15T18:41:41.801Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=eb0e51fafde3f2d89b79e0643835761438ce01ca907b85d978e5603f4cd23968 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4340224-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Feb0e51fafde3f2d89b79e0643835761438ce01ca907b85d978e5603f4cd23968
time=2026-10-15T18:44:12.376Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4340224-svc/providers/Microsoft.Resources/deployments/eb0e51fafde3f2d89b79e0643835761438ce01ca907b85d978e5603f4cd23968/operationStatuses/08584095171843738676\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4340224-svc/providers/Microsoft.Resources/deployments/cluster-s6k3y22exiy7k\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4340224-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j4340224-svc\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\",\\r\\n                \\\"message\\\": \\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\"https://ah-ci01-se-j4340224-1.vault.azure.net\\\\\\\"\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T19:40:29Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9990016-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.118:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9990016/providers/microsoft.monitor/accounts/hcps-j9990016", namespace="ocm-arohcpci01-2tfcsk6itjfjpcjhjvmmra3bj97fh52v-d7p2j6h4b2a7d8x", pod="arohcp-monitor-kube-state-metrics-7988dc648c-p222m", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tfcsk6itjfjpcjhjvmmra3bj97fh52v-d7p2j6h4b2a7d8x/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-15T19:41:30Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9990016-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.118:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9990016/providers/microsoft.monitor/accounts/hcps-j9990016", namespace="ocm-arohcpci01-2tfcsk6itjfjpcjhjvmmra3bj97fh52v-d7p2j6h4b2a7d8x", pod="arohcp-monitor-kube-state-metrics-7988dc648c-p222m", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tfcsk6itjfjpcjhjvmmra3bj97fh52v-d7p2j6h4b2a7d8x/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-rlsbjpbgfwb4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x10ca8a9c7020>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-rlsbjpbgfwb4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-rlsbjpbgfwb4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T18:35:50.9752505Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-rlsbjpbgfwb4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-rlsbjpbgfwb4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T18:35:50.9752505Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x10ca8a9c7000>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-rlsbjpbgfwb4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T18:35:50.9752505Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x10ca89c688c0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x10ca8a9c6fe0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-rlsbjpbgfwb4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T18:35:50.9752505Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x10ca8b745020>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-rlsbjpbgfwb4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x10ca8a9c7020>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-rlsbjpbgfwb4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-rlsbjpbgfwb4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T18:35:50.9752505Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-rlsbjpbgfwb4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-rlsbjpbgfwb4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T18:35:50.9752505Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x10ca8a9c7000>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-rlsbjpbgfwb4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T18:35:50.9752505Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x10ca89c688c0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x10ca8a9c6fe0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-rlsbjpbgfwb4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T18:35:50.9752505Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x10ca8b745020>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T18:54:43Z
  Ended:       2026-10-15T19:10:40Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j9483648-svc", component="backend", name="clusterresources", region="centralus", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
Firing 2:
  State:       Resolved
  Started:     2026-10-15T19:00:37Z
  Ended:       2026-10-15T19:08:45Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j9483648-svc", component="backend", name="csstatedump", region="centralus", severity="warning"
  Description: Backend controller workqueue csstatedump has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints
Unexpected error:
    <*fmt.wrapError | 0x3a4a12910c00>: 
    failed waiting for nodepool="np-1" for cluster "cluster-np-labels-taints" in resourcegroup="rg-np-labels-taints-8ks8rxhxw5sq" to finish creating: GET https://rp.j8130048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe16e54a-aadf-4a22-b30f-9bedaedbd115
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe16e54a-aadf-4a22-b30f-9bedaedbd115",
      "name": "fe16e54a-aadf-4a22-b30f-9bedaedbd115",
      "status": "Failed",
      "startTime": "2026-10-15T18:10:42.530486252Z",
      "endTime": "2026-10-15T18:39:44.318899002Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine t0d7s5o6x9h0u4c-np-1-wvgbf-9gtwm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-8ks8rxhxw5sq--managed/providers/Microsoft.Compute/virtualMachines/t0d7s5o6x9h0u4c-np-1-wvgbf-9gtwm to exist\nMachine t0d7s5o6x9h0u4c-np-1-wvgbf-wrlsl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-8ks8rxhxw5sq--managed/providers/Microsoft.Compute/virtualMachines/t0d7s5o6x9h0u4c-np-1-wvgbf-wrlsl to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine t0d7s5o6x9h0u4c-np-1-wvgbf-9gtwm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-8ks8rxhxw5sq--managed/providers/Microsoft.Compute/virtualMachines/t0d7s5o6x9h0u4c-np-1-wvgbf-9gtwm to exist\nMachine t0d7s5o6x9h0u4c-np-1-wvgbf-wrlsl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-8ks8rxhxw5sq--managed/providers/Microsoft.Compute/virtualMachines/t0d7s5o6x9h0u4c-np-1-wvgbf-wrlsl to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8130048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe16e54a-aadf-4a22-b30f-9bedaedbd115
            correlation ID: d3945582-4d5c-4e0b-8dfc-27eb4589f5aa
            client request ID: 118925eb-c455-4151-9620-b893190957ed
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints
Unexpected error:
    <*fmt.wrapError | 0x3a4a12910c00>: 
    failed waiting for nodepool="np-1" for cluster "cluster-np-labels-taints" in resourcegroup="rg-np-labels-taints-8ks8rxhxw5sq" to finish creating: GET https://rp.j8130048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe16e54a-aadf-4a22-b30f-9bedaedbd115
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe16e54a-aadf-4a22-b30f-9bedaedbd115",
      "name": "fe16e54a-aadf-4a22-b30f-9bedaedbd115",
      "status": "Failed",
      "startTime": "2026-10-15T18:10:42.530486252Z",
      "endTime": "2026-10-15T18:39:44.318899002Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine t0d7s5o6x9h0u4c-np-1-wvgbf-9gtwm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-8ks8rxhxw5sq--managed/providers/Microsoft.Compute/virtualMachines/t0d7s5o6x9h0u4c-np-1-wvgbf-9gtwm to exist\nMachine t0d7s5o6x9h0u4c-np-1-wvgbf-wrlsl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-8ks8rxhxw5sq--managed/providers/Microsoft.Compute/virtualMachines/t0d7s5o6x9h0u4c-np-1-wvgbf-wrlsl to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine t0d7s5o6x9h0u4c-np-1-wvgbf-9gtwm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-8ks8rxhxw5sq--managed/providers/Microsoft.Compute/virtualMachines/t0d7s5o6x9h0u4c-np-1-wvgbf-9gtwm to exist\nMachine t0d7s5o6x9h0u4c-np-1-wvgbf-wrlsl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-8ks8rxhxw5sq--managed/providers/Microsoft.Compute/virtualMachines/t0d7s5o6x9h0u4c-np-1-wvgbf-wrlsl to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8130048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe16e54a-aadf-4a22-b30f-9bedaedbd115
            correlation ID: d3945582-4d5c-4e0b-8dfc-27eb4589f5aa
            client request ID: 118925eb-c455-4151-9620-b893190957ed
occurred

time=2026-10-15T17:40:31.958Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description="Step cluster\n  Kind: ARM\n  Template: templates/svc-cluster.bicep\n  Parameters: configurations/svc-cluster.tmpl.bicepparam"
time=2026-10-15T17:40:33.353Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster
time=2026-10-15T17:40:34.443Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=3eb11a33c36fe8ab0e9da558ed847d75eacd0b9970bd89f40e3a3a3db9b5a15b portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0765440-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F3eb11a33c36fe8ab0e9da558ed847d75eacd0b9970bd89f40e3a3a3db9b5a15b
time=2026-10-15T17:43:04.846Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0765440-svc/providers/Microsoft.Resources/deployments/3eb11a33c36fe8ab0e9da558ed847d75eacd0b9970bd89f40e3a3a3db9b5a15b/operationStatuses/08584095208516246503\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0765440-svc/providers/Microsoft.Resources/deployments/cluster-eg5umvwrivayu\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0765440-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j0765440-svc\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\",\\r\\n                \\\"message\\\": \\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\"https://ah-ci01-se-j0765440-1.vault.azure.net\\\\\\\"\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-10-15T17:33:07.002Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1 description="Step kubelet-ds\n Kind: Helm\n"
time=2026-10-15T17:33:07.498Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1 values=map[enabled:true]
time=2026-10-15T17:33:07.498Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1
time=2026-10-15T17:33:07.598Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1 namespace=kubelet-ds
time=2026-10-15T17:33:07.719Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1 dryRun=false
time=2026-10-15T17:33:07.719Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1
time=2026-10-15T17:33:07.774Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1
time=2026-10-15T17:38:08.700Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1 err="resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 14/15\ncontext deadline exceeded"
time=2026-10-15T17:38:08.700Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1 err="stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 14/15\ncontext deadline exceeded"

alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T17:43:36Z
  Severity:    Sev3
  Labels:      alertname="MaestroAgentMetricsProxyDown", cluster="ci01-j4658048-mgmt-2", component="monitoring-infrastructure", region="centralus", severity="3"
  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.

alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T17:40:08Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4658048-mgmt-1", component="kubernetes-infrastructure", namespace="kube-system", pod="set-kubelet-parameters-for-scale-r7jxg", region="centralus", severity="warning"
  Description: Pod kube-system/set-kubelet-parameters-for-scale-r7jxg has been in a non-ready state for longer than 5 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool "np-1" for v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0x380262502ee0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "v20261001" in resourcegroup="v20261001-gvmgczj2jv9r" to finish creating: GET https://rp.j9447936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc671297-4e62-407d-b405-0c4924c5ce46
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc671297-4e62-407d-b405-0c4924c5ce46",
      "name": "fc671297-4e62-407d-b405-0c4924c5ce46",
      "status": "Failed",
      "startTime": "2026-10-15T18:15:12.704146991Z",
      "endTime": "2026-10-15T18:44:20.410859237Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9447936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc671297-4e62-407d-b405-0c4924c5ce46
            correlation ID: 619c20cd-92fd-4829-89e3-0fb066983b2c
            client request ID: 92814f76-d6eb-4996-a770-357ba0eb53c0
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool "np-1" for v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0x380262502ee0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "v20261001" in resourcegroup="v20261001-gvmgczj2jv9r" to finish creating: GET https://rp.j9447936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc671297-4e62-407d-b405-0c4924c5ce46
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc671297-4e62-407d-b405-0c4924c5ce46",
      "name": "fc671297-4e62-407d-b405-0c4924c5ce46",
      "status": "Failed",
      "startTime": "2026-10-15T18:15:12.704146991Z",
      "endTime": "2026-10-15T18:44:20.410859237Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9447936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc671297-4e62-407d-b405-0c4924c5ce46
            correlation ID: 619c20cd-92fd-4829-89e3-0fb066983b2c
            client request ID: 92814f76-d6eb-4996-a770-357ba0eb53c0
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool "np-1" for v20260901preview cluster "v20260901"
Unexpected error:
    <*fmt.wrapError | 0x16cbbb3841e0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "v20260901" in resourcegroup="v20260901preview-qc5d6jswb5qj" to finish creating: GET https://rp.j9447936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc6a40ed-7014-400d-bad8-400eddcb5f30
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc6a40ed-7014-400d-bad8-400eddcb5f30",
      "name": "dc6a40ed-7014-400d-bad8-400eddcb5f30",
      "status": "Failed",
      "startTime": "2026-10-15T18:11:06.765137924Z",
      "endTime": "2026-10-15T18:40:10.370410387Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9447936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc6a40ed-7014-400d-bad8-400eddcb5f30
            correlation ID: 36d679d4-7225-44c2-8efc-12d1c6c12f83
            client request ID: 0fff6042-c3bb-454b-84b5-80c6919f7c90
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool "np-1" for v20260901preview cluster "v20260901"
Unexpected error:
    <*fmt.wrapError | 0x16cbbb3841e0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "v20260901" in resourcegroup="v20260901preview-qc5d6jswb5qj" to finish creating: GET https://rp.j9447936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc6a40ed-7014-400d-bad8-400eddcb5f30
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc6a40ed-7014-400d-bad8-400eddcb5f30",
      "name": "dc6a40ed-7014-400d-bad8-400eddcb5f30",
      "status": "Failed",
      "startTime": "2026-10-15T18:11:06.765137924Z",
      "endTime": "2026-10-15T18:40:10.370410387Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9447936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc6a40ed-7014-400d-bad8-400eddcb5f30
            correlation ID: 36d679d4-7225-44c2-8efc-12d1c6c12f83
            client request ID: 0fff6042-c3bb-454b-84b5-80c6919f7c90
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-8shnfp
Unexpected error:
    <*fmt.wrapError | 0x272e47b96540>: 
    failed to create HCP cluster np-noedge-8shnfp: failed waiting for cluster="np-noedge-8shnfp" in resourcegroup="rg-np-noedge-8shnfp-gttkb9wtnnb4" to finish creating: GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6eb1774a-462c-4933-9f8d-664c52f9a6f9
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6eb1774a-462c-4933-9f8d-664c52f9a6f9",
      "name": "6eb1774a-462c-4933-9f8d-664c52f9a6f9",
      "status": "Failed",
      "startTime": "2026-10-15T17:57:55.947947599Z",
      "endTime": "2026-10-15T18:27:03.337200298Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service is pending"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6eb1774a-462c-4933-9f8d-664c52f9a6f9
            correlation ID: aed6b19e-e002-4183-8925-f15bb6214fc7
            client request ID: 45dff945-3cb8-4467-a1f3-78455ef2f77e
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-8shnfp
Unexpected error:
    <*fmt.wrapError | 0x272e47b96540>: 
    failed to create HCP cluster np-noedge-8shnfp: failed waiting for cluster="np-noedge-8shnfp" in resourcegroup="rg-np-noedge-8shnfp-gttkb9wtnnb4" to finish creating: GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6eb1774a-462c-4933-9f8d-664c52f9a6f9
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6eb1774a-462c-4933-9f8d-664c52f9a6f9",
      "name": "6eb1774a-462c-4933-9f8d-664c52f9a6f9",
      "status": "Failed",
      "startTime": "2026-10-15T17:57:55.947947599Z",
      "endTime": "2026-10-15T18:27:03.337200298Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service is pending"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6eb1774a-462c-4933-9f8d-664c52f9a6f9
            correlation ID: aed6b19e-e002-4183-8925-f15bb6214fc7
            client request ID: 45dff945-3cb8-4467-a1f3-78455ef2f77e
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cks5ffxqxcl6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x8639e37eb80>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cks5ffxqxcl6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cks5ffxqxcl6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T17:28:57.8726846Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cks5ffxqxcl6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cks5ffxqxcl6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T17:28:57.8726846Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x8639e37eb60>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cks5ffxqxcl6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T17:28:57.8726846Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x8639d1f4aa0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x8639e37eb40>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cks5ffxqxcl6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T17:28:57.8726846Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x8639f3c97d0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cks5ffxqxcl6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x8639e37eb80>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cks5ffxqxcl6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cks5ffxqxcl6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T17:28:57.8726846Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cks5ffxqxcl6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cks5ffxqxcl6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T17:28:57.8726846Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x8639e37eb60>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cks5ffxqxcl6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T17:28:57.8726846Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x8639d1f4aa0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x8639e37eb40>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cks5ffxqxcl6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T17:28:57.8726846Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x8639f3c97d0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster "cilium-cluster"
Unexpected error:
    <*errors.joinError | 0x25ab5d2f4048>: 
    VerifyNodesReady failed: no nodes found in the cluster
    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded
    failed to create NodePool cilium-np: failed waiting for nodepool="cilium-np" for cluster "cilium-cluster" in resourcegroup="complex-cilium-kv-f6lcns96bcwj" to finish creating: GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f76d63e-49ed-404f-b37c-48424e27353b
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f76d63e-49ed-404f-b37c-48424e27353b",
      "name": "4f76d63e-49ed-404f-b37c-48424e27353b",
      "status": "Failed",
      "startTime": "2026-10-15T17:42:06.28082824Z",
      "endTime": "2026-10-15T18:01:13.384332944Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f76d63e-49ed-404f-b37c-48424e27353b
            correlation ID: 6104ceea-1c65-4048-88e7-0a754957130d
            client request ID: a0f4a7cb-ab20-4af0-83f6-d8bc8019bf10
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster "cilium-cluster"
Unexpected error:
    <*errors.joinError | 0x25ab5d2f4048>: 
    VerifyNodesReady failed: no nodes found in the cluster
    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded
    failed to create NodePool cilium-np: failed waiting for nodepool="cilium-np" for cluster "cilium-cluster" in resourcegroup="complex-cilium-kv-f6lcns96bcwj" to finish creating: GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f76d63e-49ed-404f-b37c-48424e27353b
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f76d63e-49ed-404f-b37c-48424e27353b",
      "name": "4f76d63e-49ed-404f-b37c-48424e27353b",
      "status": "Failed",
      "startTime": "2026-10-15T17:42:06.28082824Z",
      "endTime": "2026-10-15T18:01:13.384332944Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f76d63e-49ed-404f-b37c-48424e27353b
            correlation ID: 6104ceea-1c65-4048-88e7-0a754957130d
            client request ID: a0f4a7cb-ab20-4af0-83f6-d8bc8019bf10
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:154]: failed to verify node count of 2 after deleting nodepool
Expected success, but got an error:
    <*errors.errorString | 0x922244cd220>: 
    cluster np-delete-hcp-cluster: expected 2 nodes, found 4; nodes per pool: np-deletable: [u1i7j2r4v1j1a3a-np-deletable-hjjzc-b6p7m, u1i7j2r4v1j1a3a-np-deletable-hjjzc-rbs4q], np-delete-main: [u1i7j2r4v1j1a3a-np-delete-main-8wbbv-t4dx7, u1i7j2r4v1j1a3a-np-delete-main-8wbbv-wwsbq]
    {
        s: "cluster np-delete-hcp-cluster: expected 2 nodes, found 4; nodes per pool: np-deletable: [u1i7j2r4v1j1a3a-np-deletable-hjjzc-b6p7m, u1i7j2r4v1j1a3a-np-deletable-hjjzc-rbs4q], np-delete-main: [u1i7j2r4v1j1a3a-np-delete-main-8wbbv-t4dx7, u1i7j2r4v1j1a3a-np-delete-main-8wbbv-wwsbq]",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:154]: failed to verify node count of 2 after deleting nodepool
Expected success, but got an error:
    <*errors.errorString | 0x922244cd220>: 
    cluster np-delete-hcp-cluster: expected 2 nodes, found 4; nodes per pool: np-deletable: [u1i7j2r4v1j1a3a-np-deletable-hjjzc-b6p7m, u1i7j2r4v1j1a3a-np-deletable-hjjzc-rbs4q], np-delete-main: [u1i7j2r4v1j1a3a-np-delete-main-8wbbv-t4dx7, u1i7j2r4v1j1a3a-np-delete-main-8wbbv-wwsbq]
    {
        s: "cluster np-delete-hcp-cluster: expected 2 nodes, found 4; nodes per pool: np-deletable: [u1i7j2r4v1j1a3a-np-deletable-hjjzc-b6p7m, u1i7j2r4v1j1a3a-np-deletable-hjjzc-rbs4q], np-delete-main: [u1i7j2r4v1j1a3a-np-delete-main-8wbbv-t4dx7, u1i7j2r4v1j1a3a-np-delete-main-8wbbv-wwsbq]",
    }

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:
    <*errors.joinError | 0x2d72cc8aa180>: 
    resource group "autoscaling-cluster-7hjmhzg78tkl": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="autoscaling-hcp-cluster" in resourcegroup="autoscaling-cluster-7hjmhzg78tkl" to finish deleting: GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c3202962-b736-4f51-840b-2c3b32fd9172
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c3202962-b736-4f51-840b-2c3b32fd9172",
      "name": "c3202962-b736-4f51-840b-2c3b32fd9172",
      "status": "Failed",
      "startTime": "2026-10-15T18:00:50.353777Z",
      "endTime": "2026-10-15T18:24:53.714360344Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tfbvc1dj03nufhiu0qumf4o5mmkbqp1 still exists (deletion dispatched at 2026-10-15T18:00:50Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c3202962-b736-4f51-840b-2c3b32fd9172
            correlation ID: 2ab924cd-c6a5-4535-afa1-4285e0d7421a
            client request ID: 05581b93-1450-43ab-a2a8-826d6a1faeda
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:
    <*errors.joinError | 0x2d72cc8aa180>: 
    resource group "autoscaling-cluster-7hjmhzg78tkl": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="autoscaling-hcp-cluster" in resourcegroup="autoscaling-cluster-7hjmhzg78tkl" to finish deleting: GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c3202962-b736-4f51-840b-2c3b32fd9172
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c3202962-b736-4f51-840b-2c3b32fd9172",
      "name": "c3202962-b736-4f51-840b-2c3b32fd9172",
      "status": "Failed",
      "startTime": "2026-10-15T18:00:50.353777Z",
      "endTime": "2026-10-15T18:24:53.714360344Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tfbvc1dj03nufhiu0qumf4o5mmkbqp1 still exists (deletion dispatched at 2026-10-15T18:00:50Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c3202962-b736-4f51-840b-2c3b32fd9172
            correlation ID: 2ab924cd-c6a5-4535-afa1-4285e0d7421a
            client request ID: 05581b93-1450-43ab-a2a8-826d6a1faeda
occurred

alert fired 4 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T18:44:11Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j3520000-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cluster-api", endpoint="http", environment="ci01", instance="10.128.64.223:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3520000/providers/microsoft.monitor/accounts/hcps-j3520000", namespace="ocm-arohcpci01-2tfbvc1dj03nufhiu0qumf4o5mmkbqp1-w6h1y1s8u1u3w6q", pod="arohcp-monitor-kube-state-metrics-7988dc648c-5k8tw", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tfbvc1dj03nufhiu0qumf4o5mmkbqp1-w6h1y1s8u1u3w6q/cluster-api has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-15T18:44:11Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j3520000-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.223:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3520000/providers/microsoft.monitor/accounts/hcps-j3520000", namespace="ocm-arohcpci01-2tfbvc1dj03nufhiu0qumf4o5mmkbqp1-w6h1y1s8u1u3w6q", pod="arohcp-monitor-kube-state-metrics-7988dc648c-5k8tw", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tfbvc1dj03nufhiu0qumf4o5mmkbqp1-w6h1y1s8u1u3w6q/capi-provider has not matched the expected number of replicas for longer than 30 minutes.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-15T18:44:11Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j3520000-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.223:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3520000/providers/microsoft.monitor/accounts/hcps-j3520000", namespace="ocm-arohcpci01-2tfbvc1dj03nufhiu0qumf4o5mmkbqp1-w6h1y1s8u1u3w6q", pod="arohcp-monitor-kube-state-metrics-7988dc648c-5k8tw", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tfbvc1dj03nufhiu0qumf4o5mmkbqp1-w6h1y1s8u1u3w6q/capi-provider has not matched the expected number of replicas for longer than 30 minutes.
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-15T18:44:11Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j3520000-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cluster-api", endpoint="http", environment="ci01", instance="10.128.64.223:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3520000/providers/microsoft.monitor/accounts/hcps-j3520000", namespace="ocm-arohcpci01-2tfbvc1dj03nufhiu0qumf4o5mmkbqp1-w6h1y1s8u1u3w6q", pod="arohcp-monitor-kube-state-metrics-7988dc648c-5k8tw", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tfbvc1dj03nufhiu0qumf4o5mmkbqp1-w6h1y1s8u1u3w6q/cluster-api has not matched the expected number of replicas for longer than 30 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-6xdv46kj2t6c/cluster-candidate-4-23-ds6g6p should provision
Unexpected error:
    <*fmt.wrapError | 0x100c616f6120>: 
    failed to create HCP cluster cluster-candidate-4-23-ds6g6p: failed waiting for cluster="cluster-candidate-4-23-ds6g6p" in resourcegroup="rg-candidate-4-23-6xdv46kj2t6c" to finish creating: GET https://rp.j0840320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f51531d5-d416-4611-a4d7-f0f2d7392927
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f51531d5-d416-4611-a4d7-f0f2d7392927",
      "name": "f51531d5-d416-4611-a4d7-f0f2d7392927",
      "status": "Failed",
      "startTime": "2026-10-15T17:08:33.97123397Z",
      "endTime": "2026-10-15T17:37:33.800671635Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0840320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f51531d5-d416-4611-a4d7-f0f2d7392927
            correlation ID: b723ee7e-f41a-48af-8379-6cf9b8adcb05
            client request ID: a255119f-4ee0-4d8e-8cd6-e4350ddd6c3d
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-6xdv46kj2t6c/cluster-candidate-4-23-ds6g6p should provision
Unexpected error:
    <*fmt.wrapError | 0x100c616f6120>: 
    failed to create HCP cluster cluster-candidate-4-23-ds6g6p: failed waiting for cluster="cluster-candidate-4-23-ds6g6p" in resourcegroup="rg-candidate-4-23-6xdv46kj2t6c" to finish creating: GET https://rp.j0840320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f51531d5-d416-4611-a4d7-f0f2d7392927
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f51531d5-d416-4611-a4d7-f0f2d7392927",
      "name": "f51531d5-d416-4611-a4d7-f0f2d7392927",
      "status": "Failed",
      "startTime": "2026-10-15T17:08:33.97123397Z",
      "endTime": "2026-10-15T17:37:33.800671635Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0840320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f51531d5-d416-4611-a4d7-f0f2d7392927
            correlation ID: b723ee7e-f41a-48af-8379-6cf9b8adcb05
            client request ID: a255119f-4ee0-4d8e-8cd6-e4350ddd6c3d
occurred

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T17:45:04Z
  Ended:       2026-10-15T17:58:01Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j0840320-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.184:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0840320/providers/microsoft.monitor/accounts/hcps-j0840320", namespace="ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j", pod="arohcp-monitor-kube-state-metrics-7988dc648c-5z7k7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j/capi-provider has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-15T17:46:01Z
  Ended:       2026-10-15T17:57:03Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j0840320-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.184:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0840320/providers/microsoft.monitor/accounts/hcps-j0840320", namespace="ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j", pod="arohcp-monitor-kube-state-metrics-7988dc648c-5z7k7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j/capi-provider has not matched the expected number of replicas for longer than 30 minutes.

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T17:33:59Z
  Ended:       2026-10-15T18:00:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j0840320-mgmt-2", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.64.184:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0840320/providers/microsoft.monitor/accounts/hcps-j0840320", namespace="ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j", pod="etcd-1", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="69abfa76-e157-4c11-8f36-2d5d03c40035"
  Description: Pod ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j/etcd-1 (etcd) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Resolved
  Started:     2026-10-15T17:33:59Z
  Ended:       2026-10-15T17:59:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j0840320-mgmt-2", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.64.184:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0840320/providers/microsoft.monitor/accounts/hcps-j0840320", namespace="ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j", pod="etcd-1", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="69abfa76-e157-4c11-8f36-2d5d03c40035"
  Description: Pod ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j/etcd-1 (etcd) is in waiting state (reason: "CrashLoopBackOff").

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T17:31:05Z
  Ended:       2026-10-15T17:56:03Z
  Severity:    Sev3
  Labels:      alertname="KubeStatefulSetReplicasMismatch", cluster="ci01-j0840320-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.184:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0840320/providers/microsoft.monitor/accounts/hcps-j0840320", namespace="ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j", pod="arohcp-monitor-kube-state-metrics-7988dc648c-5z7k7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", statefulset="etcd"
  Description: StatefulSet ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j/etcd has not matched the expected number of replicas for longer than 15 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-15T17:31:05Z
  Ended:       2026-10-15T18:01:04Z
  Severity:    Sev3
  Labels:      alertname="KubeStatefulSetReplicasMismatch", cluster="ci01-j0840320-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.184:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0840320/providers/microsoft.monitor/accounts/hcps-j0840320", namespace="ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j", pod="arohcp-monitor-kube-state-metrics-7988dc648c-5z7k7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", statefulset="etcd"
  Description: StatefulSet ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j/etcd has not matched the expected number of replicas for longer than 15 minutes.

time=2026-10-15T16:46:56.692Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description="Step cluster\n  Kind: ARM\n  Template: templates/svc-cluster.bicep\n  Parameters: configurations/svc-cluster.tmpl.bicepparam"
time=2026-10-15T16:46:58.243Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster
time=2026-10-15T16:46:59.266Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=e4fad1d34566ab93ca0eb67800d6c1fe07b016fafb66268a48e93221f4f410ee portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6980864-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fe4fad1d34566ab93ca0eb67800d6c1fe07b016fafb66268a48e93221f4f410ee
time=2026-10-15T16:49:29.880Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6980864-svc/providers/Microsoft.Resources/deployments/e4fad1d34566ab93ca0eb67800d6c1fe07b016fafb66268a48e93221f4f410ee/operationStatuses/08584095240667656957\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6980864-svc/providers/Microsoft.Resources/deployments/cluster-uuk7cfdwkwfeq\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6980864-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j6980864-svc\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\",\\r\\n                \\\"message\\\": \\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\"https://ah-ci01-se-j6980864-1.vault.azure.net\\\\\\\"\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k7swxgrgxs2p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2d6557bb26c0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k7swxgrgxs2p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k7swxgrgxs2p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T17:02:54.4463595Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k7swxgrgxs2p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k7swxgrgxs2p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T17:02:54.4463595Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x2d6557bb26a0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k7swxgrgxs2p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T17:02:54.4463595Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x2d655713e140>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x2d6557bb2680>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k7swxgrgxs2p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T17:02:54.4463595Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x2d6558c091d0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k7swxgrgxs2p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2d6557bb26c0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k7swxgrgxs2p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k7swxgrgxs2p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T17:02:54.4463595Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k7swxgrgxs2p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k7swxgrgxs2p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T17:02:54.4463595Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x2d6557bb26a0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k7swxgrgxs2p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T17:02:54.4463595Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x2d655713e140>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x2d6557bb2680>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k7swxgrgxs2p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T17:02:54.4463595Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x2d6558c091d0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T18:05:49Z
  Severity:    Sev3
  Labels:      alertname="LeaderElectionLeaseStale", cluster="ci01-j7905664-mgmt-1", component="leader-election", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.65.54:8080", job="kube-state-metrics", lease="work-manager", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7905664/providers/microsoft.monitor/accounts/services-j7905664", namespace="klusterlet-2tfbjf51so3bj30gni3l73dohrjlrc07", pod="arohcp-monitor-kube-state-metrics-7988dc648c-mz4bk", prometheus="prometheus/prometheus", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Leader election lease work-manager in namespace klusterlet-2tfbjf51so3bj30gni3l73dohrjlrc07 on cluster ci01-j7905664-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.

alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T18:00:18Z
  Severity:    Sev3
  Labels:      alertname="LeaderElectionLeaseStale", cluster="ci01-j3950080-mgmt-1", component="leader-election", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.234:8080", job="kube-state-metrics", lease="work-manager", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3950080/providers/microsoft.monitor/accounts/services-j3950080", namespace="klusterlet-2tfbf4h0d9nqbkbie85eg76q9fu1hc63", pod="arohcp-monitor-kube-state-metrics-7988dc648c-zrvv8", prometheus="prometheus/prometheus", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Leader election lease work-manager in namespace klusterlet-2tfbf4h0d9nqbkbie85eg76q9fu1hc63 on cluster ci01-j3950080-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T17:36:21Z
  Ended:       2026-10-15T17:42:24Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3115648-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfbijr2kab846rrhmkesku5ajemuk8m-j0f0h0p4l0h6p9i", pod="azure-disk-csi-driver-controller-b9bc7746d-g7xbc", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfbijr2kab846rrhmkesku5ajemuk8m-j0f0h0p4l0h6p9i/azure-disk-csi-driver-controller-b9bc7746d-g7xbc has been in a non-ready state for longer than 5 minutes.

time=2026-10-15T16:33:01.433Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description="Step cluster\n  Kind: ARM\n  Template: templates/svc-cluster.bicep\n  Parameters: configurations/svc-cluster.tmpl.bicepparam"
time=2026-10-15T16:33:03.008Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster
time=2026-10-15T16:33:06.001Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=4a75caebbcb0957378a554574a53ab5c38ad767d8dfe35c65ac1690bd2903591 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5342592-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F4a75caebbcb0957378a554574a53ab5c38ad767d8dfe35c65ac1690bd2903591
time=2026-10-15T17:03:01.188Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"

alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T17:00:33Z
  Severity:    Sev3
  Labels:      alertname="MaestroAgentMetricsProxyDown", cluster="ci01-j5342592-mgmt-2", component="monitoring-infrastructure", region="westus3", severity="3"
  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-15T17:00:33Z
  Severity:    Sev3
  Labels:      alertname="MaestroAgentMetricsProxyDown", cluster="ci01-j5342592-mgmt-1", component="monitoring-infrastructure", region="westus3", severity="3"
  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T17:09:40Z
  Ended:       2026-10-15T17:15:40Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerRetryHotLoop", cluster="ci01-j1515264-svc", component="backend", name="operationclusterupdate", region="centralus", severity="warning"
  Description: Backend controller workqueue operationclusterupdate has a retry ratio of > 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:
    <*errors.joinError | 0xb39be402570>: 
    resource group "rg-zstream-upgrade-4-22-qkz2pwtj5n7j": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-zstream-4-22-9x7njw" in resourcegroup="rg-zstream-upgrade-4-22-qkz2pwtj5n7j" to finish deleting: GET https://rp.j0325120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d1c8a5-60e3-4c19-acaf-41b8b35cb86a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d1c8a5-60e3-4c19-acaf-41b8b35cb86a",
      "name": "a5d1c8a5-60e3-4c19-acaf-41b8b35cb86a",
      "status": "Failed",
      "startTime": "2026-10-15T17:05:09.736854856Z",
      "endTime": "2026-10-15T17:29:18.684928477Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tfb63s18q5cesobi99a7b7bb7k3a72n still exists (deletion dispatched at 2026-10-15T17:05:10Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0325120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d1c8a5-60e3-4c19-acaf-41b8b35cb86a
            correlation ID: ce6f6715-7b02-4412-b5e3-4934b1a27812
            client request ID: bf6cd98a-47f4-43d5-af65-2ee26c5ddfb6
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:
    <*errors.joinError | 0xb39be402570>: 
    resource group "rg-zstream-upgrade-4-22-qkz2pwtj5n7j": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-zstream-4-22-9x7njw" in resourcegroup="rg-zstream-upgrade-4-22-qkz2pwtj5n7j" to finish deleting: GET https://rp.j0325120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d1c8a5-60e3-4c19-acaf-41b8b35cb86a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d1c8a5-60e3-4c19-acaf-41b8b35cb86a",
      "name": "a5d1c8a5-60e3-4c19-acaf-41b8b35cb86a",
      "status": "Failed",
      "startTime": "2026-10-15T17:05:09.736854856Z",
      "endTime": "2026-10-15T17:29:18.684928477Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tfb63s18q5cesobi99a7b7bb7k3a72n still exists (deletion dispatched at 2026-10-15T17:05:10Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0325120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d1c8a5-60e3-4c19-acaf-41b8b35cb86a
            correlation ID: ce6f6715-7b02-4412-b5e3-4934b1a27812
            client request ID: bf6cd98a-47f4-43d5-af65-2ee26c5ddfb6
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:123]: Timed out after 600.001s.
Expected success, but got an error:
    <*errors.errorString | 0xc8e80bea4a0>: 
    must-gather CLI test failures:
      query-basic: command failed: exit status 1
    output: �[37m[17:06:46.277]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-basic-824225575"
    }�[0m
    �[37m[17:06:50.310]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:06:48.7179236Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\", ActivityId:\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\", SubActivityId:\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\"}, IsPermanent:false})",
      "name": "detailedInfraOrchestrationLogs"
    }�[0m
    �[37m[17:06:50.310]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "query \"backendControllerConditions\" time window 78/289 failed (KQL: let _startTime = datetime(2026-10-14T23:31:46.2770422Z);\nlet _endTime = datetime(2026-10-14T23:36:46.2770421Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled",
      "name": "backendControllerConditions"
    }�[0m
    �[37m[17:06:50.376]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{
      "input": "/tmp/must-gather-cli-e2e-query-basic-824225575/custom/custom-query_cosmosResourceSnapshots.jsonl",
      "output": "/tmp/must-gather-cli-e2e-query-basic-824225575/cosmosContent"
    }�[0m
    �[37m[17:06:50.376]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-basic-824225575/custom/custom-query_cosmosResourceSnapshots.jsonl"
    }�[0m
    �[37m[17:06:50.697]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{
      "count": 1106,
      "files": 1
    }�[0m
    �[37m[17:06:50.697]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-basic-824225575/cosmosContent"
    }�[0m
    �[37m[17:06:50.706]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m
    �[37m[17:07:03.236]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{
      "commits": 613
    }�[0m
    �[37m[17:07:03.237]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{
      "clusterIDs": null,
      "output": "/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect"
    }�[0m
    �[37m[17:07:03.356]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{
      "clusterIDs": [
        "2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
      ]
    }�[0m
    �[37m[17:07:03.470]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{
      "clusterNames": [
        "ci01-j0325120-mgmt-2",
        "ci01-j0325120-svc"
      ]
    }�[0m
    �[37m[17:07:03.503]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespaces": [
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
      ]
    }�[0m
    �[37m[17:07:03.503]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{
      "managementCluster": "ci01-j0325120-mgmt-2",
      "namespaceCount": 2
    }�[0m
    �[37m[17:07:03.503]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
      "output": "/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
    }�[0m
    �[37m[17:07:03.650]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v",
      "output": "/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
    }�[0m
    �[37m[17:07:04.108]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:06:48.7179236Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\", ActivityId:\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\", SubActivityId:\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\"}, IsPermanent:false})",
      "output": "/tmp/must-gather-cli-e2e-query-basic-824225575"
    }�[0m
    �[37m[17:07:04.108]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:06:48.7179236Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\", ActivityId:\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\", SubActivityId:\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\"}, IsPermanent:false})"
    }�[0m
    
      query-skip-hcp-logs: command failed: exit status 1
    output: �[37m[17:07:04.396]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816"
    }�[0m
    �[37m[17:07:13.034]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:07:11.3621455Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\", ActivityId:\"2d73786e-0dab-4734-916c-277b95dfe6b3\", SubActivityId:\"2d73786e-0dab-4734-916c-277b95dfe6b3\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"2d73786e-0dab-4734-916c-277b95dfe6b3\", ActivityStack:\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\"}, IsPermanent:false})",
      "name": "detailedInfraOrchestrationLogs"
    }�[0m
    �[37m[17:07:13.034]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "query \"backendControllerConditions\" time window 83/289 failed (KQL: let _startTime = datetime(2026-10-14T23:57:04.3961354Z);\nlet _endTime = datetime(2026-10-15T00:02:04.3961353Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled",
      "name": "backendControllerConditions"
    }�[0m
    �[37m[17:07:13.071]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{
      "input": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/custom/custom-query_cosmosResourceSnapshots.jsonl",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/cosmosContent"
    }�[0m
    �[37m[17:07:13.072]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/custom/custom-query_cosmosResourceSnapshots.jsonl"
    }�[0m
    �[37m[17:07:13.393]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{
      "count": 1106,
      "files": 1
    }�[0m
    �[37m[17:07:13.393]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/cosmosContent"
    }�[0m
    �[37m[17:07:13.403]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m
    �[37m[17:07:25.804]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{
      "commits": 613
    }�[0m
    �[37m[17:07:25.805]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{
      "clusterIDs": null,
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect"
    }�[0m
    �[37m[17:07:25.912]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{
      "clusterIDs": [
        "2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
      ]
    }�[0m
    �[37m[17:07:26.021]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{
      "clusterNames": [
        "ci01-j0325120-mgmt-2",
        "ci01-j0325120-svc"
      ]
    }�[0m
    �[37m[17:07:26.141]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespaces": [
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
      ]
    }�[0m
    �[37m[17:07:26.141]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{
      "managementCluster": "ci01-j0325120-mgmt-2",
      "namespaceCount": 2
    }�[0m
    �[37m[17:07:26.141]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
    }�[0m
    �[37m[17:07:26.291]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
    }�[0m
    �[37m[17:07:26.780]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:07:11.3621455Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\", ActivityId:\"2d73786e-0dab-4734-916c-277b95dfe6b3\", SubActivityId:\"2d73786e-0dab-4734-916c-277b95dfe6b3\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"2d73786e-0dab-4734-916c-277b95dfe6b3\", ActivityStack:\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\"}, IsPermanent:false})",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816"
    }�[0m
    �[37m[17:07:26.780]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:07:11.3621455Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\", ActivityId:\"2d73786e-0dab-4734-916c-277b95dfe6b3\", SubActivityId:\"2d73786e-0dab-4734-916c-277b95dfe6b3\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"2d73786e-0dab-4734-916c-277b95dfe6b3\", ActivityStack:\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\"}, IsPermanent:false})"
    }�[0m
    
      query-skip-kubernetes-events: command failed: exit status 1
    output: �[37m[17:07:27.073]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188"
    }�[0m
    �[37m[17:07:30.812]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:07:29.3103489Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\", ActivityId:\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\", SubActivityId:\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\", ActivityStack:\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\"}, IsPermanent:false})",
      "name": "detailedInfraOrchestrationLogs"
    }�[0m
    �[37m[17:07:30.812]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "query \"backendControllerConditions\" time window 76/289 failed (KQL: let _startTime = datetime(2026-10-14T23:22:27.0728956Z);\nlet _endTime = datetime(2026-10-14T23:27:27.0728955Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-14T23:22:27.0728956Z);\nlet _endTime = datetime(2026-10-14T23:27:27.0728955Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context canceled",
      "name": "backendControllerConditions"
    }�[0m
    �[37m[17:07:30.813]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{
      "input": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/custom/custom-query_cosmosResourceSnapshots.jsonl",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/cosmosContent"
    }�[0m
    �[37m[17:07:30.813]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/custom/custom-query_cosmosResourceSnapshots.jsonl"
    }�[0m
    �[37m[17:07:31.138]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{
      "count": 1106,
      "files": 1
    }�[0m
    �[37m[17:07:31.138]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/cosmosContent"
    }�[0m
    �[37m[17:07:31.148]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m
    �[37m[17:07:42.926]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{
      "clusterIDs": null,
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect"
    }�[0m
    �[37m[17:07:43.042]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{
      "clusterIDs": [
        "2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
      ]
    }�[0m
    �[37m[17:07:43.166]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{
      "clusterNames": [
        "ci01-j0325120-mgmt-2",
        "ci01-j0325120-svc"
      ]
    }�[0m
    �[37m[17:07:43.202]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespaces": [
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
      ]
    }�[0m
    �[37m[17:07:43.202]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{
      "managementCluster": "ci01-j0325120-mgmt-2",
      "namespaceCount": 2
    }�[0m
    �[37m[17:07:43.202]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
    }�[0m
    �[37m[17:07:43.435]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
    }�[0m
    �[37m[17:07:43.942]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:07:29.3103489Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\", ActivityId:\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\", SubActivityId:\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\", ActivityStack:\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\"}, IsPermanent:false})\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\n",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188"
    }�[0m
    �[37m[17:07:43.942]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:07:29.3103489Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\", ActivityId:\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\", SubActivityId:\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\", ActivityStack:\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\"}, IsPermanent:false})\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\n"
    }�[0m
    
      query-collect-systemd-logs: command failed: exit status 1
    output: �[37m[17:07:44.226]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200"
    }�[0m
    �[37m[17:07:47.896]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:07:46.3200405Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\", ActivityId:\"66a3669f-0e2c-4860-86dc-f849756f632e\", SubActivityId:\"66a3669f-0e2c-4860-86dc-f849756f632e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"66a3669f-0e2c-4860-86dc-f849756f632e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\"}, IsPermanent:false})",
      "name": "detailedInfraOrchestrationLogs"
    }�[0m
    �[37m[17:07:47.896]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "query \"backendControllerConditions\" time window 76/289 failed (KQL: let _startTime = datetime(2026-10-14T23:22:44.2253704Z);\nlet _endTime = datetime(2026-10-14T23:27:44.2253703Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-14T23:22:44.2253704Z);\nlet _endTime = datetime(2026-10-14T23:27:44.2253703Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context canceled",
      "name": "backendControllerConditions"
    }�[0m
    �[37m[17:07:47.968]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{
      "input": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/custom/custom-query_cosmosResourceSnapshots.jsonl",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/cosmosContent"
    }�[0m
    �[37m[17:07:47.968]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/custom/custom-query_cosmosResourceSnapshots.jsonl"
    }�[0m
    �[37m[17:07:48.283]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{
      "count": 1106,
      "files": 1
    }�[0m
    �[37m[17:07:48.283]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/cosmosContent"
    }�[0m
    �[37m[17:07:48.293]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m
    �[37m[17:08:00.494]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{
      "commits": 613
    }�[0m
    �[37m[17:08:00.495]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{
      "clusterIDs": null,
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect"
    }�[0m
    �[37m[17:08:00.610]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{
      "clusterIDs": [
        "2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
      ]
    }�[0m
    �[37m[17:08:00.715]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{
      "clusterNames": [
        "ci01-j0325120-mgmt-2",
        "ci01-j0325120-svc"
      ]
    }�[0m
    �[37m[17:08:00.747]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespaces": [
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
      ]
    }�[0m
    �[37m[17:08:00.747]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{
      "managementCluster": "ci01-j0325120-mgmt-2",
      "namespaceCount": 2
    }�[0m
    �[37m[17:08:00.747]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
    }�[0m
    �[37m[17:08:01.068]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
    }�[0m
    �[37m[17:08:01.547]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:07:46.3200405Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\", ActivityId:\"66a3669f-0e2c-4860-86dc-f849756f632e\", SubActivityId:\"66a3669f-0e2c-4860-86dc-f849756f632e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"66a3669f-0e2c-4860-86dc-f849756f632e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\"}, IsPermanent:false})",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200"
    }�[0m
    �[37m[17:08:01.548]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:07:46.3200405Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\", ActivityId:\"66a3669f-0e2c-4860-86dc-f849756f632e\", SubActivityId:\"66a3669f-0e2c-4860-86dc-f849756f632e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"66a3669f-0e2c-4860-86dc-f849756f632e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\"}, IsPermanent:false})"
    }�[0m
    
      query-skip-hcp-and-events: command failed: exit status 1
    output: �[37m[17:08:01.848]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533"
    }�[0m
    �[37m[17:08:05.427]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:08:03.7881995Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\", ActivityId:\"d19bda2c-8f67-41fb-bf81-27155da79740\", SubActivityId:\"d19bda2c-8f67-41fb-bf81-27155da79740\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"d19bda2c-8f67-41fb-bf81-27155da79740\", ActivityStack:\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\"}, IsPermanent:false})",
      "name": "detailedInfraOrchestrationLogs"
    }�[0m
    �[37m[17:08:05.428]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "query \"backendControllerConditions\" time window 85/289 failed (KQL: let _startTime = datetime(2026-10-15T00:08:01.8473531Z);\nlet _endTime = datetime(2026-10-15T00:13:01.8473530Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j0325120-mgmt-2', 'ci01-j0325120-svc', 'ci01-j0325120-mgmt-2')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-15T00:08:01.8473531Z);\nlet _endTime = datetime(2026-10-15T00:13:01.8473530Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j0325120-mgmt-2', 'ci01-j0325120-svc', 'ci01-j0325120-mgmt-2')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context canceled",
      "name": "backendControllerConditions"
    }�[0m
    �[37m[17:08:05.428]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{
      "input": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/custom/custom-query_cosmosResourceSnapshots.jsonl",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/cosmosContent"
    }�[0m
    �[37m[17:08:05.428]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/custom/custom-query_cosmosResourceSnapshots.jsonl"
    }�[0m
    �[37m[17:08:05.743]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{
      "count": 1106,
      "files": 1
    }�[0m
    �[37m[17:08:05.743]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/cosmosContent"
    }�[0m
    �[37m[17:08:05.752]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m
    �[37m[17:08:15.101]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{
      "clusterIDs": null,
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect"
    }�[0m
    �[37m[17:08:15.221]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{
      "clusterIDs": [
        "2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
      ]
    }�[0m
    �[37m[17:08:15.326]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{
      "clusterNames": [
        "ci01-j0325120-mgmt-2",
        "ci01-j0325120-svc"
      ]
    }�[0m
    �[37m[17:08:15.357]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespaces": [
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
      ]
    }�[0m
    �[37m[17:08:15.357]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{
      "managementCluster": "ci01-j0325120-mgmt-2",
      "namespaceCount": 2
    }�[0m
    �[37m[17:08:15.357]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
    }�[0m
    �[37m[17:08:15.495]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
    }�[0m
    �[37m[17:08:15.976]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:08:03.7881995Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\", ActivityId:\"d19bda2c-8f67-41fb-bf81-27155da79740\", SubActivityId:\"d19bda2c-8f67-41fb-bf81-27155da79740\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"d19bda2c-8f67-41fb-bf81-27155da79740\", ActivityStack:\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\"}, IsPermanent:false})\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\n",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533"
    }�[0m
    �[37m[17:08:15.977]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:08:03.7881995Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\", ActivityId:\"d19bda2c-8f67-41fb-bf81-27155da79740\", SubActivityId:\"d19bda2c-8f67-41fb-bf81-27155da79740\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"d19bda2c-8f67-41fb-bf81-27155da79740\", ActivityStack:\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\"}, IsPermanent:false})\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\n"
    }�[0m
    
    {
        s: "must-gather CLI test failures:\n  query-basic: command failed: exit status 1\noutput: \x1b[37m[17:06:46.277]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575\"\n}\x1b[0m\n\x1b[37m[17:06:50.310]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:06:48.7179236Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\\\", ActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", SubActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \\u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\\\"}, IsPermanent:false})\",\n  \"name\": \"detailedInfraOrchestrationLogs\"\n}\x1b[0m\n\x1b[37m[17:06:50.310]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"query \\\"backendControllerConditions\\\" time window 78/289 failed (KQL: let _startTime = datetime(2026-10-14T23:31:46.2770422Z);\\nlet _endTime = datetime(2026-10-14T23:36:46.2770421Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n  \"name\": \"backendControllerConditions\"\n}\x1b[0m\n\x1b[37m[17:06:50.376]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgenerating Cosmos content Git history\x1b[0m \x1b[90m{\n  \"input\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:06:50.376]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mParsing log file\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n}\x1b[0m\n\x1b[37m[17:06:50.697]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mFound data dump entries\x1b[0m \x1b[90m{\n  \"count\": 1106,\n  \"files\": 1\n}\x1b[0m\n\x1b[37m[17:06:50.697]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mInitializing git repository\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:06:50.706]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mProcessing entries and creating commits\x1b[0m \x1b[90m{}\x1b[0m\n\x1b[37m[17:07:03.236]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mCompleted\x1b[0m \x1b[90m{\n  \"commits\": 613\n}\x1b[0m\n\x1b[37m[17:07:03.237]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for clusters in the resource group\x1b[0m \x1b[90m{\n  \"clusterIDs\": null,\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect\"\n}\x1b[0m\n\x1b[37m[17:07:03.356]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: cluster IDs to inspect\x1b[0m \x1b[90m{\n  \"clusterIDs\": [\n    \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n  ]\n}\x1b[0m\n\x1b[37m[17:07:03.470]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered infra cluster names\x1b[0m \x1b[90m{\n  \"clusterNames\": [\n    \"ci01-j0325120-mgmt-2\",\n    \"ci01-j0325120-svc\"\n  ]\n}\x1b[0m\n\x1b[37m[17:07:03.503]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaces\": [\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n  ]\n}\x1b[0m\n\x1b[37m[17:07:03.503]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\x1b[0m \x1b[90m{\n  \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaceCount\": 2\n}\x1b[0m\n\x1b[37m[17:07:03.503]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n}\x1b[0m\n\x1b[37m[17:07:03.650]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n}\x1b[0m\n\x1b[37m[17:07:04.108]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mmust-gather finished with errors; partial content may have been written\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:06:48.7179236Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\\\", ActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", SubActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \\u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\\\"}, IsPermanent:false})\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575\"\n}\x1b[0m\n\x1b[37m[17:07:04.108]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mcommand failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:06:48.7179236Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\\\", ActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", SubActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \\u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\\\"}, IsPermanent:false})\"\n}\x1b[0m\n\n  query-skip-hcp-logs: command failed: exit status 1\noutput: \x1b[37m[17:07:04.396]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816\"\n}\x1b[0m\n\x1b[37m[17:07:13.034]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:11.3621455Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\\\", ActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", SubActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \\u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\\\"}, IsPermanent:false})\",\n  \"name\": \"detailedInfraOrchestrationLogs\"\n}\x1b[0m\n\x1b[37m[17:07:13.034]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"query \\\"backendControllerConditions\\\" time window 83/289 failed (KQL: let _startTime = datetime(2026-10-14T23:57:04.3961354Z);\\nlet _endTime = datetime(2026-10-15T00:02:04.3961353Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n  \"name\": \"backendControllerConditions\"\n}\x1b[0m\n\x1b[37m[17:07:13.071]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgenerating Cosmos content Git history\x1b[0m \x1b[90m{\n  \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:07:13.072]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mParsing log file\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n}\x1b[0m\n\x1b[37m[17:07:13.393]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mFound data dump entries\x1b[0m \x1b[90m{\n  \"count\": 1106,\n  \"files\": 1\n}\x1b[0m\n\x1b[37m[17:07:13.393]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mInitializing git repository\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:07:13.403]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mProcessing entries and creating commits\x1b[0m \x1b[90m{}\x1b[0m\n\x1b[37m[17:07:25.804]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mCompleted\x1b[0m \x1b[90m{\n  \"commits\": 613\n}\x1b[0m\n\x1b[37m[17:07:25.805]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for clusters in the resource group\x1b[0m \x1b[90m{\n  \"clusterIDs\": null,\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect\"\n}\x1b[0m\n\x1b[37m[17:07:25.912]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: cluster IDs to inspect\x1b[0m \x1b[90m{\n  \"clusterIDs\": [\n    \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n  ]\n}\x1b[0m\n\x1b[37m[17:07:26.021]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered infra cluster names\x1b[0m \x1b[90m{\n  \"clusterNames\": [\n    \"ci01-j0325120-mgmt-2\",\n    \"ci01-j0325120-svc\"\n  ]\n}\x1b[0m\n\x1b[37m[17:07:26.141]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaces\": [\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n  ]\n}\x1b[0m\n\x1b[37m[17:07:26.141]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\x1b[0m \x1b[90m{\n  \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaceCount\": 2\n}\x1b[0m\n\x1b[37m[17:07:26.141]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n}\x1b[0m\n\x1b[37m[17:07:26.291]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n}\x1b[0m\n\x1b[37m[17:07:26.780]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mmust-gather finished with errors; partial content may have been written\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:11.3621455Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\\\", ActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", SubActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \\u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\\\"}, IsPermanent:false})\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816\"\n}\x1b[0m\n\x1b[37m[17:07:26.780]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mcommand failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:11.3621455Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\\\", ActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", SubActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \\u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\\\"}, IsPermanent:false})\"\n}\x1b[0m\n\n  query-skip-kubernetes-events: command failed: exit status 1\noutput: \x1b[37m[17:07:27.073]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188\"\n}\x1b[0m\n\x1b[37m[17:07:30.812]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:29.3103489Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\\\", ActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", SubActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \\u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\\\"}, IsPermanent:false})\",\n  \"name\": \"detailedInfraOrchestrationLogs\"\n}\x1b[0m\n\x1b[37m[17:07:30.812]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"query \\\"backendControllerConditions\\\" time window 76/289 failed (KQL: let _startTime = datetime(2026-10-14T23:22:27.0728956Z);\\nlet _endTime = datetime(2026-10-14T23:27:27.0728955Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-14T23:22:27.0728956Z);\\nlet _endTime = datetime(2026-10-14T23:27:27.0728955Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n  \"name\": \"backendControllerConditions\"\n}\x1b[0m\n\x1b[37m[17:07:30.813]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgenerating Cosmos content Git history\x1b[0m \x1b[90m{\n  \"input\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:07:30.813]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mParsing log file\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n}\x1b[0m\n\x1b[37m[17:07:31.138]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mFound data dump entries\x1b[0m \x1b[90m{\n  \"count\": 1106,\n  \"files\": 1\n}\x1b[0m\n\x1b[37m[17:07:31.138]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mInitializing git repository\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:07:31.148]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mProcessing entries and creating commits\x1b[0m \x1b[90m{}\x1b[0m\n\x1b[37m[17:07:42.926]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for clusters in the resource group\x1b[0m \x1b[90m{\n  \"clusterIDs\": null,\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect\"\n}\x1b[0m\n\x1b[37m[17:07:43.042]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: cluster IDs to inspect\x1b[0m \x1b[90m{\n  \"clusterIDs\": [\n    \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n  ]\n}\x1b[0m\n\x1b[37m[17:07:43.166]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered infra cluster names\x1b[0m \x1b[90m{\n  \"clusterNames\": [\n    \"ci01-j0325120-mgmt-2\",\n    \"ci01-j0325120-svc\"\n  ]\n}\x1b[0m\n\x1b[37m[17:07:43.202]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaces\": [\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n  ]\n}\x1b[0m\n\x1b[37m[17:07:43.202]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\x1b[0m \x1b[90m{\n  \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaceCount\": 2\n}\x1b[0m\n\x1b[37m[17:07:43.202]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n}\x1b[0m\n\x1b[37m[17:07:43.435]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n}\x1b[0m\n\x1b[37m[17:07:43.942]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mmust-gather finished with errors; partial content may have been written\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:29.3103489Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\\\", ActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", SubActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \\u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188\"\n}\x1b[0m\n\x1b[37m[17:07:43.942]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mcommand failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:29.3103489Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\\\", ActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", SubActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \\u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n}\x1b[0m\n\n  query-collect-systemd-logs: command failed: exit status 1\noutput: \x1b[37m[17:07:44.226]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200\"\n}\x1b[0m\n\x1b[37m[17:07:47.896]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:46.3200405Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\\\", ActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", SubActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \\u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\\\"}, IsPermanent:false})\",\n  \"name\": \"detailedInfraOrchestrationLogs\"\n}\x1b[0m\n\x1b[37m[17:07:47.896]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"query \\\"backendControllerConditions\\\" time window 76/289 failed (KQL: let _startTime = datetime(2026-10-14T23:22:44.2253704Z);\\nlet _endTime = datetime(2026-10-14T23:27:44.2253703Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-14T23:22:44.2253704Z);\\nlet _endTime = datetime(2026-10-14T23:27:44.2253703Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n  \"name\": \"backendControllerConditions\"\n}\x1b[0m\n\x1b[37m[17:07:47.968]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgenerating Cosmos content Git history\x1b[0m \x1b[90m{\n  \"input\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:07:47.968]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mParsing log file\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n}\x1b[0m\n\x1b[37m[17:07:48.283]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mFound data dump entries\x1b[0m \x1b[90m{\n  \"count\": 1106,\n  \"files\": 1\n}\x1b[0m\n\x1b[37m[17:07:48.283]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mInitializing git repository\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:07:48.293]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mProcessing entries and creating commits\x1b[0m \x1b[90m{}\x1b[0m\n\x1b[37m[17:08:00.494]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mCompleted\x1b[0m \x1b[90m{\n  \"commits\": 613\n}\x1b[0m\n\x1b[37m[17:08:00.495]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for clusters in the resource group\x1b[0m \x1b[90m{\n  \"clusterIDs\": null,\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect\"\n}\x1b[0m\n\x1b[37m[17:08:00.610]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: cluster IDs to inspect\x1b[0m \x1b[90m{\n  \"clusterIDs\": [\n    \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n  ]\n}\x1b[0m\n\x1b[37m[17:08:00.715]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered infra cluster names\x1b[0m \x1b[90m{\n  \"clusterNames\": [\n    \"ci01-j0325120-mgmt-2\",\n    \"ci01-j0325120-svc\"\n  ]\n}\x1b[0m\n\x1b[37m[17:08:00.747]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaces\": [\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n  ]\n}\x1b[0m\n\x1b[37m[17:08:00.747]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\x1b[0m \x1b[90m{\n  \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaceCount\": 2\n}\x1b[0m\n\x1b[37m[17:08:00.747]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n}\x1b[0m\n\x1b[37m[17:08:01.068]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n}\x1b[0m\n\x1b[37m[17:08:01.547]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mmust-gather finished with errors; partial content may have been written\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:46.3200405Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\\\", ActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", SubActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \\u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\\\"}, IsPermanent:false})\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200\"\n}\x1b[0m\n\x1b[37m[17:08:01.548]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mcommand failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:46.3200405Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\\\", ActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", SubActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \\u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\\\"}, IsPermanent:false})\"\n}\x1b[0m\n\n  query-skip-hcp-and-events: command failed: exit status 1\noutput: \x1b[37m[17:08:01.848]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533\"\n}\x1b[0m\n\x1b[37m[17:08:05.427]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:08:03.7881995Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\\\", ActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", SubActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \\u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\\\"}, IsPermanent:false})\",\n  \"name\": \"detailedInfraOrchestrationLogs\"\n}\x1b[0m\n\x1b[37m[17:08:05.428]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"query \\\"backendControllerConditions\\\" time window 85/289 failed (KQL: let _startTime = datetime(2026-10-15T00:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T00:13:01.8473530Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-mgmt-2', 'ci01-j0325120-svc', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-15T00:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T00:13:01.8473530Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-mgmt-2', 'ci01-j0325120-svc', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n  \"name\": \"backendControllerConditions\"\n}\x1b[0m\n\x1b[37m[17:08:05.428]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgenerating Cosmos content Git history\x1b[0m \x1b[90m{\n  \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:08:05.428]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mParsing log file\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n}\x1b[0m\n\x1b[37m[17:08:05.743]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mFound data dump entries\x1b[0m \x1b[90m{\n  \"count\": 1106,\n  \"files\": 1\n}\x1b[0m\n\x1b[37m[17:08:05.743]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mInitializing git repository\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:08:05.752]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mProcessing entries and creating commits\x1b[0m \x1b[90m{}\x1b[0m\n\x1b[37m[17:08:15.101]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for clusters in the resource group\x1b[0m \x1b[90m{\n  \"clusterIDs\": null,\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect\"\n}\x1b[0m\n\x1b[37m[17:08:15.221]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: cluster IDs to inspect\x1b[0m \x1b[90m{\n  \"clusterIDs\": [\n    \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n  ]\n}\x1b[0m\n\x1b[37m[17:08:15.326]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered infra cluster names\x1b[0m \x1b[90m{\n  \"clusterNames\": [\n    \"ci01-j0325120-mgmt-2\",\n    \"ci01-j0325120-svc\"\n  ]\n}\x1b[0m\n\x1b[37m[17:08:15.357]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaces\": [\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n  ]\n}\x1b[0m\n\x1b[37m[17:08:15.357]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\x1b[0m \x1b[90m{\n  \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaceCount\": 2\n}\x1b[0m\n\x1b[37m[17:08:15.357]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n}\x1b[0m\n\x1b[37m[17:08:15.495]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n}\x1b[0m\n\x1b[37m[17:08:15.976]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mmust-gather finished with errors; partial content may have been written\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:08:03.7881995Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\\\", ActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", SubActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \\u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533\"\n}\x1b[0m\n\x1b[37m[17:08:15.977]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mcommand failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:08:03.7881995Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\\\", ActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", SubActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \\u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n}\x1b[0m\n",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:123]: Timed out after 600.001s.
Expected success, but got an error:
    <*errors.errorString | 0xc8e80bea4a0>: 
    must-gather CLI test failures:
      query-basic: command failed: exit status 1
    output: �[37m[17:06:46.277]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-basic-824225575"
    }�[0m
    �[37m[17:06:50.310]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:06:48.7179236Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\", ActivityId:\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\", SubActivityId:\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\"}, IsPermanent:false})",
      "name": "detailedInfraOrchestrationLogs"
    }�[0m
    �[37m[17:06:50.310]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "query \"backendControllerConditions\" time window 78/289 failed (KQL: let _startTime = datetime(2026-10-14T23:31:46.2770422Z);\nlet _endTime = datetime(2026-10-14T23:36:46.2770421Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled",
      "name": "backendControllerConditions"
    }�[0m
    �[37m[17:06:50.376]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{
      "input": "/tmp/must-gather-cli-e2e-query-basic-824225575/custom/custom-query_cosmosResourceSnapshots.jsonl",
      "output": "/tmp/must-gather-cli-e2e-query-basic-824225575/cosmosContent"
    }�[0m
    �[37m[17:06:50.376]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-basic-824225575/custom/custom-query_cosmosResourceSnapshots.jsonl"
    }�[0m
    �[37m[17:06:50.697]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{
      "count": 1106,
      "files": 1
    }�[0m
    �[37m[17:06:50.697]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-basic-824225575/cosmosContent"
    }�[0m
    �[37m[17:06:50.706]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m
    �[37m[17:07:03.236]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{
      "commits": 613
    }�[0m
    �[37m[17:07:03.237]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{
      "clusterIDs": null,
      "output": "/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect"
    }�[0m
    �[37m[17:07:03.356]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{
      "clusterIDs": [
        "2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
      ]
    }�[0m
    �[37m[17:07:03.470]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{
      "clusterNames": [
        "ci01-j0325120-mgmt-2",
        "ci01-j0325120-svc"
      ]
    }�[0m
    �[37m[17:07:03.503]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespaces": [
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
      ]
    }�[0m
    �[37m[17:07:03.503]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{
      "managementCluster": "ci01-j0325120-mgmt-2",
      "namespaceCount": 2
    }�[0m
    �[37m[17:07:03.503]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
      "output": "/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
    }�[0m
    �[37m[17:07:03.650]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v",
      "output": "/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
    }�[0m
    �[37m[17:07:04.108]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:06:48.7179236Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\", ActivityId:\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\", SubActivityId:\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\"}, IsPermanent:false})",
      "output": "/tmp/must-gather-cli-e2e-query-basic-824225575"
    }�[0m
    �[37m[17:07:04.108]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:06:48.7179236Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\", ActivityId:\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\", SubActivityId:\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\"}, IsPermanent:false})"
    }�[0m
    
      query-skip-hcp-logs: command failed: exit status 1
    output: �[37m[17:07:04.396]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816"
    }�[0m
    �[37m[17:07:13.034]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:07:11.3621455Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\", ActivityId:\"2d73786e-0dab-4734-916c-277b95dfe6b3\", SubActivityId:\"2d73786e-0dab-4734-916c-277b95dfe6b3\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"2d73786e-0dab-4734-916c-277b95dfe6b3\", ActivityStack:\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\"}, IsPermanent:false})",
      "name": "detailedInfraOrchestrationLogs"
    }�[0m
    �[37m[17:07:13.034]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "query \"backendControllerConditions\" time window 83/289 failed (KQL: let _startTime = datetime(2026-10-14T23:57:04.3961354Z);\nlet _endTime = datetime(2026-10-15T00:02:04.3961353Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled",
      "name": "backendControllerConditions"
    }�[0m
    �[37m[17:07:13.071]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{
      "input": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/custom/custom-query_cosmosResourceSnapshots.jsonl",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/cosmosContent"
    }�[0m
    �[37m[17:07:13.072]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/custom/custom-query_cosmosResourceSnapshots.jsonl"
    }�[0m
    �[37m[17:07:13.393]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{
      "count": 1106,
      "files": 1
    }�[0m
    �[37m[17:07:13.393]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/cosmosContent"
    }�[0m
    �[37m[17:07:13.403]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m
    �[37m[17:07:25.804]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{
      "commits": 613
    }�[0m
    �[37m[17:07:25.805]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{
      "clusterIDs": null,
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect"
    }�[0m
    �[37m[17:07:25.912]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{
      "clusterIDs": [
        "2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
      ]
    }�[0m
    �[37m[17:07:26.021]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{
      "clusterNames": [
        "ci01-j0325120-mgmt-2",
        "ci01-j0325120-svc"
      ]
    }�[0m
    �[37m[17:07:26.141]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespaces": [
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
      ]
    }�[0m
    �[37m[17:07:26.141]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{
      "managementCluster": "ci01-j0325120-mgmt-2",
      "namespaceCount": 2
    }�[0m
    �[37m[17:07:26.141]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
    }�[0m
    �[37m[17:07:26.291]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
    }�[0m
    �[37m[17:07:26.780]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:07:11.3621455Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\", ActivityId:\"2d73786e-0dab-4734-916c-277b95dfe6b3\", SubActivityId:\"2d73786e-0dab-4734-916c-277b95dfe6b3\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"2d73786e-0dab-4734-916c-277b95dfe6b3\", ActivityStack:\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\"}, IsPermanent:false})",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816"
    }�[0m
    �[37m[17:07:26.780]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:07:11.3621455Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\", ActivityId:\"2d73786e-0dab-4734-916c-277b95dfe6b3\", SubActivityId:\"2d73786e-0dab-4734-916c-277b95dfe6b3\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"2d73786e-0dab-4734-916c-277b95dfe6b3\", ActivityStack:\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\"}, IsPermanent:false})"
    }�[0m
    
      query-skip-kubernetes-events: command failed: exit status 1
    output: �[37m[17:07:27.073]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188"
    }�[0m
    �[37m[17:07:30.812]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:07:29.3103489Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\", ActivityId:\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\", SubActivityId:\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\", ActivityStack:\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\"}, IsPermanent:false})",
      "name": "detailedInfraOrchestrationLogs"
    }�[0m
    �[37m[17:07:30.812]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "query \"backendControllerConditions\" time window 76/289 failed (KQL: let _startTime = datetime(2026-10-14T23:22:27.0728956Z);\nlet _endTime = datetime(2026-10-14T23:27:27.0728955Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-14T23:22:27.0728956Z);\nlet _endTime = datetime(2026-10-14T23:27:27.0728955Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context canceled",
      "name": "backendControllerConditions"
    }�[0m
    �[37m[17:07:30.813]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{
      "input": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/custom/custom-query_cosmosResourceSnapshots.jsonl",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/cosmosContent"
    }�[0m
    �[37m[17:07:30.813]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/custom/custom-query_cosmosResourceSnapshots.jsonl"
    }�[0m
    �[37m[17:07:31.138]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{
      "count": 1106,
      "files": 1
    }�[0m
    �[37m[17:07:31.138]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/cosmosContent"
    }�[0m
    �[37m[17:07:31.148]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m
    �[37m[17:07:42.926]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{
      "clusterIDs": null,
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect"
    }�[0m
    �[37m[17:07:43.042]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{
      "clusterIDs": [
        "2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
      ]
    }�[0m
    �[37m[17:07:43.166]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{
      "clusterNames": [
        "ci01-j0325120-mgmt-2",
        "ci01-j0325120-svc"
      ]
    }�[0m
    �[37m[17:07:43.202]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespaces": [
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
      ]
    }�[0m
    �[37m[17:07:43.202]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{
      "managementCluster": "ci01-j0325120-mgmt-2",
      "namespaceCount": 2
    }�[0m
    �[37m[17:07:43.202]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
    }�[0m
    �[37m[17:07:43.435]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
    }�[0m
    �[37m[17:07:43.942]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:07:29.3103489Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\", ActivityId:\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\", SubActivityId:\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\", ActivityStack:\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\"}, IsPermanent:false})\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\n",
      "output": "/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188"
    }�[0m
    �[37m[17:07:43.942]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:07:29.3103489Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\", ActivityId:\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\", SubActivityId:\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\", ActivityStack:\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\"}, IsPermanent:false})\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\n"
    }�[0m
    
      query-collect-systemd-logs: command failed: exit status 1
    output: �[37m[17:07:44.226]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200"
    }�[0m
    �[37m[17:07:47.896]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:07:46.3200405Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\", ActivityId:\"66a3669f-0e2c-4860-86dc-f849756f632e\", SubActivityId:\"66a3669f-0e2c-4860-86dc-f849756f632e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"66a3669f-0e2c-4860-86dc-f849756f632e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\"}, IsPermanent:false})",
      "name": "detailedInfraOrchestrationLogs"
    }�[0m
    �[37m[17:07:47.896]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "query \"backendControllerConditions\" time window 76/289 failed (KQL: let _startTime = datetime(2026-10-14T23:22:44.2253704Z);\nlet _endTime = datetime(2026-10-14T23:27:44.2253703Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-14T23:22:44.2253704Z);\nlet _endTime = datetime(2026-10-14T23:27:44.2253703Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context canceled",
      "name": "backendControllerConditions"
    }�[0m
    �[37m[17:07:47.968]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{
      "input": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/custom/custom-query_cosmosResourceSnapshots.jsonl",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/cosmosContent"
    }�[0m
    �[37m[17:07:47.968]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/custom/custom-query_cosmosResourceSnapshots.jsonl"
    }�[0m
    �[37m[17:07:48.283]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{
      "count": 1106,
      "files": 1
    }�[0m
    �[37m[17:07:48.283]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/cosmosContent"
    }�[0m
    �[37m[17:07:48.293]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m
    �[37m[17:08:00.494]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{
      "commits": 613
    }�[0m
    �[37m[17:08:00.495]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{
      "clusterIDs": null,
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect"
    }�[0m
    �[37m[17:08:00.610]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{
      "clusterIDs": [
        "2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
      ]
    }�[0m
    �[37m[17:08:00.715]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{
      "clusterNames": [
        "ci01-j0325120-mgmt-2",
        "ci01-j0325120-svc"
      ]
    }�[0m
    �[37m[17:08:00.747]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespaces": [
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
      ]
    }�[0m
    �[37m[17:08:00.747]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{
      "managementCluster": "ci01-j0325120-mgmt-2",
      "namespaceCount": 2
    }�[0m
    �[37m[17:08:00.747]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
    }�[0m
    �[37m[17:08:01.068]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
    }�[0m
    �[37m[17:08:01.547]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:07:46.3200405Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\", ActivityId:\"66a3669f-0e2c-4860-86dc-f849756f632e\", SubActivityId:\"66a3669f-0e2c-4860-86dc-f849756f632e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"66a3669f-0e2c-4860-86dc-f849756f632e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\"}, IsPermanent:false})",
      "output": "/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200"
    }�[0m
    �[37m[17:08:01.548]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:07:46.3200405Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\", ActivityId:\"66a3669f-0e2c-4860-86dc-f849756f632e\", SubActivityId:\"66a3669f-0e2c-4860-86dc-f849756f632e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"66a3669f-0e2c-4860-86dc-f849756f632e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\"}, IsPermanent:false})"
    }�[0m
    
      query-skip-hcp-and-events: command failed: exit status 1
    output: �[37m[17:08:01.848]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533"
    }�[0m
    �[37m[17:08:05.427]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:08:03.7881995Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\", ActivityId:\"d19bda2c-8f67-41fb-bf81-27155da79740\", SubActivityId:\"d19bda2c-8f67-41fb-bf81-27155da79740\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"d19bda2c-8f67-41fb-bf81-27155da79740\", ActivityStack:\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\"}, IsPermanent:false})",
      "name": "detailedInfraOrchestrationLogs"
    }�[0m
    �[37m[17:08:05.428]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{
      "err": "query \"backendControllerConditions\" time window 85/289 failed (KQL: let _startTime = datetime(2026-10-15T00:08:01.8473531Z);\nlet _endTime = datetime(2026-10-15T00:13:01.8473530Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j0325120-mgmt-2', 'ci01-j0325120-svc', 'ci01-j0325120-mgmt-2')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-15T00:08:01.8473531Z);\nlet _endTime = datetime(2026-10-15T00:13:01.8473530Z);\ndatabase('ServiceLogs').table('backendLogs')\n| where timestamp between (_startTime .. _endTime)\n| where cluster has_any ('ci01-j0325120-mgmt-2', 'ci01-j0325120-svc', 'ci01-j0325120-mgmt-2')\n| where container_name == 'aro-hcp-backend'\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\n| extend controller_name = extract(\"/hcpOpenShiftControllers/([^\\\\/]+)\", 1, tostring(log.currentResourceID))\n| mv-expand condition = log.content.properties.status.conditions\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context canceled",
      "name": "backendControllerConditions"
    }�[0m
    �[37m[17:08:05.428]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{
      "input": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/custom/custom-query_cosmosResourceSnapshots.jsonl",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/cosmosContent"
    }�[0m
    �[37m[17:08:05.428]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/custom/custom-query_cosmosResourceSnapshots.jsonl"
    }�[0m
    �[37m[17:08:05.743]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{
      "count": 1106,
      "files": 1
    }�[0m
    �[37m[17:08:05.743]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{
      "path": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/cosmosContent"
    }�[0m
    �[37m[17:08:05.752]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m
    �[37m[17:08:15.101]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{
      "clusterIDs": null,
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect"
    }�[0m
    �[37m[17:08:15.221]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{
      "clusterIDs": [
        "2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
      ]
    }�[0m
    �[37m[17:08:15.326]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{
      "clusterNames": [
        "ci01-j0325120-mgmt-2",
        "ci01-j0325120-svc"
      ]
    }�[0m
    �[37m[17:08:15.357]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespaces": [
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
        "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
      ]
    }�[0m
    �[37m[17:08:15.357]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{
      "managementCluster": "ci01-j0325120-mgmt-2",
      "namespaceCount": 2
    }�[0m
    �[37m[17:08:15.357]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q"
    }�[0m
    �[37m[17:08:15.495]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{
      "cluster": "ci01-j0325120-mgmt-2",
      "namespace": "ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v"
    }�[0m
    �[37m[17:08:15.976]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:08:03.7881995Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\", ActivityId:\"d19bda2c-8f67-41fb-bf81-27155da79740\", SubActivityId:\"d19bda2c-8f67-41fb-bf81-27155da79740\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"d19bda2c-8f67-41fb-bf81-27155da79740\", ActivityStack:\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\"}, IsPermanent:false})\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\n",
      "output": "/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533"
    }�[0m
    �[37m[17:08:15.977]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{
      "err": "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:08:03.7881995Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\", ActivityId:\"d19bda2c-8f67-41fb-bf81-27155da79740\", SubActivityId:\"d19bda2c-8f67-41fb-bf81-27155da79740\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"d19bda2c-8f67-41fb-bf81-27155da79740\", ActivityStack:\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\"}, IsPermanent:false})\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\n"
    }�[0m
    
    {
        s: "must-gather CLI test failures:\n  query-basic: command failed: exit status 1\noutput: \x1b[37m[17:06:46.277]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575\"\n}\x1b[0m\n\x1b[37m[17:06:50.310]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:06:48.7179236Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\\\", ActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", SubActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \\u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\\\"}, IsPermanent:false})\",\n  \"name\": \"detailedInfraOrchestrationLogs\"\n}\x1b[0m\n\x1b[37m[17:06:50.310]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"query \\\"backendControllerConditions\\\" time window 78/289 failed (KQL: let _startTime = datetime(2026-10-14T23:31:46.2770422Z);\\nlet _endTime = datetime(2026-10-14T23:36:46.2770421Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n  \"name\": \"backendControllerConditions\"\n}\x1b[0m\n\x1b[37m[17:06:50.376]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgenerating Cosmos content Git history\x1b[0m \x1b[90m{\n  \"input\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:06:50.376]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mParsing log file\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n}\x1b[0m\n\x1b[37m[17:06:50.697]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mFound data dump entries\x1b[0m \x1b[90m{\n  \"count\": 1106,\n  \"files\": 1\n}\x1b[0m\n\x1b[37m[17:06:50.697]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mInitializing git repository\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:06:50.706]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mProcessing entries and creating commits\x1b[0m \x1b[90m{}\x1b[0m\n\x1b[37m[17:07:03.236]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mCompleted\x1b[0m \x1b[90m{\n  \"commits\": 613\n}\x1b[0m\n\x1b[37m[17:07:03.237]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for clusters in the resource group\x1b[0m \x1b[90m{\n  \"clusterIDs\": null,\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect\"\n}\x1b[0m\n\x1b[37m[17:07:03.356]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: cluster IDs to inspect\x1b[0m \x1b[90m{\n  \"clusterIDs\": [\n    \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n  ]\n}\x1b[0m\n\x1b[37m[17:07:03.470]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered infra cluster names\x1b[0m \x1b[90m{\n  \"clusterNames\": [\n    \"ci01-j0325120-mgmt-2\",\n    \"ci01-j0325120-svc\"\n  ]\n}\x1b[0m\n\x1b[37m[17:07:03.503]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaces\": [\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n  ]\n}\x1b[0m\n\x1b[37m[17:07:03.503]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\x1b[0m \x1b[90m{\n  \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaceCount\": 2\n}\x1b[0m\n\x1b[37m[17:07:03.503]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n}\x1b[0m\n\x1b[37m[17:07:03.650]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n}\x1b[0m\n\x1b[37m[17:07:04.108]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mmust-gather finished with errors; partial content may have been written\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:06:48.7179236Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\\\", ActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", SubActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \\u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\\\"}, IsPermanent:false})\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575\"\n}\x1b[0m\n\x1b[37m[17:07:04.108]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mcommand failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:06:48.7179236Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\\\", ActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", SubActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \\u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\\\"}, IsPermanent:false})\"\n}\x1b[0m\n\n  query-skip-hcp-logs: command failed: exit status 1\noutput: \x1b[37m[17:07:04.396]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816\"\n}\x1b[0m\n\x1b[37m[17:07:13.034]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:11.3621455Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\\\", ActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", SubActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \\u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\\\"}, IsPermanent:false})\",\n  \"name\": \"detailedInfraOrchestrationLogs\"\n}\x1b[0m\n\x1b[37m[17:07:13.034]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"query \\\"backendControllerConditions\\\" time window 83/289 failed (KQL: let _startTime = datetime(2026-10-14T23:57:04.3961354Z);\\nlet _endTime = datetime(2026-10-15T00:02:04.3961353Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n  \"name\": \"backendControllerConditions\"\n}\x1b[0m\n\x1b[37m[17:07:13.071]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgenerating Cosmos content Git history\x1b[0m \x1b[90m{\n  \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:07:13.072]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mParsing log file\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n}\x1b[0m\n\x1b[37m[17:07:13.393]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mFound data dump entries\x1b[0m \x1b[90m{\n  \"count\": 1106,\n  \"files\": 1\n}\x1b[0m\n\x1b[37m[17:07:13.393]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mInitializing git repository\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:07:13.403]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mProcessing entries and creating commits\x1b[0m \x1b[90m{}\x1b[0m\n\x1b[37m[17:07:25.804]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mCompleted\x1b[0m \x1b[90m{\n  \"commits\": 613\n}\x1b[0m\n\x1b[37m[17:07:25.805]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for clusters in the resource group\x1b[0m \x1b[90m{\n  \"clusterIDs\": null,\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect\"\n}\x1b[0m\n\x1b[37m[17:07:25.912]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: cluster IDs to inspect\x1b[0m \x1b[90m{\n  \"clusterIDs\": [\n    \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n  ]\n}\x1b[0m\n\x1b[37m[17:07:26.021]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered infra cluster names\x1b[0m \x1b[90m{\n  \"clusterNames\": [\n    \"ci01-j0325120-mgmt-2\",\n    \"ci01-j0325120-svc\"\n  ]\n}\x1b[0m\n\x1b[37m[17:07:26.141]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaces\": [\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n  ]\n}\x1b[0m\n\x1b[37m[17:07:26.141]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\x1b[0m \x1b[90m{\n  \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaceCount\": 2\n}\x1b[0m\n\x1b[37m[17:07:26.141]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n}\x1b[0m\n\x1b[37m[17:07:26.291]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n}\x1b[0m\n\x1b[37m[17:07:26.780]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mmust-gather finished with errors; partial content may have been written\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:11.3621455Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\\\", ActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", SubActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \\u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\\\"}, IsPermanent:false})\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816\"\n}\x1b[0m\n\x1b[37m[17:07:26.780]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mcommand failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:11.3621455Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\\\", ActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", SubActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \\u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\\\"}, IsPermanent:false})\"\n}\x1b[0m\n\n  query-skip-kubernetes-events: command failed: exit status 1\noutput: \x1b[37m[17:07:27.073]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188\"\n}\x1b[0m\n\x1b[37m[17:07:30.812]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:29.3103489Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\\\", ActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", SubActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \\u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\\\"}, IsPermanent:false})\",\n  \"name\": \"detailedInfraOrchestrationLogs\"\n}\x1b[0m\n\x1b[37m[17:07:30.812]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"query \\\"backendControllerConditions\\\" time window 76/289 failed (KQL: let _startTime = datetime(2026-10-14T23:22:27.0728956Z);\\nlet _endTime = datetime(2026-10-14T23:27:27.0728955Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-14T23:22:27.0728956Z);\\nlet _endTime = datetime(2026-10-14T23:27:27.0728955Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n  \"name\": \"backendControllerConditions\"\n}\x1b[0m\n\x1b[37m[17:07:30.813]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgenerating Cosmos content Git history\x1b[0m \x1b[90m{\n  \"input\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:07:30.813]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mParsing log file\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n}\x1b[0m\n\x1b[37m[17:07:31.138]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mFound data dump entries\x1b[0m \x1b[90m{\n  \"count\": 1106,\n  \"files\": 1\n}\x1b[0m\n\x1b[37m[17:07:31.138]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mInitializing git repository\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:07:31.148]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mProcessing entries and creating commits\x1b[0m \x1b[90m{}\x1b[0m\n\x1b[37m[17:07:42.926]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for clusters in the resource group\x1b[0m \x1b[90m{\n  \"clusterIDs\": null,\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect\"\n}\x1b[0m\n\x1b[37m[17:07:43.042]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: cluster IDs to inspect\x1b[0m \x1b[90m{\n  \"clusterIDs\": [\n    \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n  ]\n}\x1b[0m\n\x1b[37m[17:07:43.166]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered infra cluster names\x1b[0m \x1b[90m{\n  \"clusterNames\": [\n    \"ci01-j0325120-mgmt-2\",\n    \"ci01-j0325120-svc\"\n  ]\n}\x1b[0m\n\x1b[37m[17:07:43.202]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaces\": [\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n  ]\n}\x1b[0m\n\x1b[37m[17:07:43.202]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\x1b[0m \x1b[90m{\n  \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaceCount\": 2\n}\x1b[0m\n\x1b[37m[17:07:43.202]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n}\x1b[0m\n\x1b[37m[17:07:43.435]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n}\x1b[0m\n\x1b[37m[17:07:43.942]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mmust-gather finished with errors; partial content may have been written\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:29.3103489Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\\\", ActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", SubActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \\u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188\"\n}\x1b[0m\n\x1b[37m[17:07:43.942]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mcommand failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:29.3103489Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\\\", ActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", SubActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \\u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n}\x1b[0m\n\n  query-collect-systemd-logs: command failed: exit status 1\noutput: \x1b[37m[17:07:44.226]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200\"\n}\x1b[0m\n\x1b[37m[17:07:47.896]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:46.3200405Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\\\", ActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", SubActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \\u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\\\"}, IsPermanent:false})\",\n  \"name\": \"detailedInfraOrchestrationLogs\"\n}\x1b[0m\n\x1b[37m[17:07:47.896]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"query \\\"backendControllerConditions\\\" time window 76/289 failed (KQL: let _startTime = datetime(2026-10-14T23:22:44.2253704Z);\\nlet _endTime = datetime(2026-10-14T23:27:44.2253703Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-14T23:22:44.2253704Z);\\nlet _endTime = datetime(2026-10-14T23:27:44.2253703Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n  \"name\": \"backendControllerConditions\"\n}\x1b[0m\n\x1b[37m[17:07:47.968]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgenerating Cosmos content Git history\x1b[0m \x1b[90m{\n  \"input\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:07:47.968]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mParsing log file\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n}\x1b[0m\n\x1b[37m[17:07:48.283]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mFound data dump entries\x1b[0m \x1b[90m{\n  \"count\": 1106,\n  \"files\": 1\n}\x1b[0m\n\x1b[37m[17:07:48.283]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mInitializing git repository\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:07:48.293]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mProcessing entries and creating commits\x1b[0m \x1b[90m{}\x1b[0m\n\x1b[37m[17:08:00.494]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mCompleted\x1b[0m \x1b[90m{\n  \"commits\": 613\n}\x1b[0m\n\x1b[37m[17:08:00.495]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for clusters in the resource group\x1b[0m \x1b[90m{\n  \"clusterIDs\": null,\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect\"\n}\x1b[0m\n\x1b[37m[17:08:00.610]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: cluster IDs to inspect\x1b[0m \x1b[90m{\n  \"clusterIDs\": [\n    \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n  ]\n}\x1b[0m\n\x1b[37m[17:08:00.715]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered infra cluster names\x1b[0m \x1b[90m{\n  \"clusterNames\": [\n    \"ci01-j0325120-mgmt-2\",\n    \"ci01-j0325120-svc\"\n  ]\n}\x1b[0m\n\x1b[37m[17:08:00.747]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaces\": [\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n  ]\n}\x1b[0m\n\x1b[37m[17:08:00.747]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\x1b[0m \x1b[90m{\n  \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaceCount\": 2\n}\x1b[0m\n\x1b[37m[17:08:00.747]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n}\x1b[0m\n\x1b[37m[17:08:01.068]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n}\x1b[0m\n\x1b[37m[17:08:01.547]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mmust-gather finished with errors; partial content may have been written\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:46.3200405Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\\\", ActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", SubActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \\u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\\\"}, IsPermanent:false})\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200\"\n}\x1b[0m\n\x1b[37m[17:08:01.548]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mcommand failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:46.3200405Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\\\", ActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", SubActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \\u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\\\"}, IsPermanent:false})\"\n}\x1b[0m\n\n  query-skip-hcp-and-events: command failed: exit status 1\noutput: \x1b[37m[17:08:01.848]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgathering must-gather logs\x1b[0m \x1b[90m{\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533\"\n}\x1b[0m\n\x1b[37m[17:08:05.427]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:08:03.7881995Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\\\", ActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", SubActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \\u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\\\"}, IsPermanent:false})\",\n  \"name\": \"detailedInfraOrchestrationLogs\"\n}\x1b[0m\n\x1b[37m[17:08:05.428]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mQuery failed\x1b[0m \x1b[90m{\n  \"err\": \"query \\\"backendControllerConditions\\\" time window 85/289 failed (KQL: let _startTime = datetime(2026-10-15T00:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T00:13:01.8473530Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-mgmt-2', 'ci01-j0325120-svc', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-15T00:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T00:13:01.8473530Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-mgmt-2', 'ci01-j0325120-svc', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n  \"name\": \"backendControllerConditions\"\n}\x1b[0m\n\x1b[37m[17:08:05.428]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mgenerating Cosmos content Git history\x1b[0m \x1b[90m{\n  \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:08:05.428]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mParsing log file\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n}\x1b[0m\n\x1b[37m[17:08:05.743]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mFound data dump entries\x1b[0m \x1b[90m{\n  \"count\": 1106,\n  \"files\": 1\n}\x1b[0m\n\x1b[37m[17:08:05.743]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mInitializing git repository\x1b[0m \x1b[90m{\n  \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/cosmosContent\"\n}\x1b[0m\n\x1b[37m[17:08:05.752]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mProcessing entries and creating commits\x1b[0m \x1b[90m{}\x1b[0m\n\x1b[37m[17:08:15.101]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for clusters in the resource group\x1b[0m \x1b[90m{\n  \"clusterIDs\": null,\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect\"\n}\x1b[0m\n\x1b[37m[17:08:15.221]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: cluster IDs to inspect\x1b[0m \x1b[90m{\n  \"clusterIDs\": [\n    \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n  ]\n}\x1b[0m\n\x1b[37m[17:08:15.326]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered infra cluster names\x1b[0m \x1b[90m{\n  \"clusterNames\": [\n    \"ci01-j0325120-mgmt-2\",\n    \"ci01-j0325120-svc\"\n  ]\n}\x1b[0m\n\x1b[37m[17:08:15.357]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaces\": [\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n    \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n  ]\n}\x1b[0m\n\x1b[37m[17:08:15.357]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\x1b[0m \x1b[90m{\n  \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespaceCount\": 2\n}\x1b[0m\n\x1b[37m[17:08:15.357]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n}\x1b[0m\n\x1b[37m[17:08:15.495]\x1b[0m \x1b[36mINFO:\x1b[0m \x1b[97mrunning oc-adm-inspect for namespace\x1b[0m \x1b[90m{\n  \"cluster\": \"ci01-j0325120-mgmt-2\",\n  \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n}\x1b[0m\n\x1b[37m[17:08:15.976]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mmust-gather finished with errors; partial content may have been written\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:08:03.7881995Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\\\", ActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", SubActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \\u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n  \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533\"\n}\x1b[0m\n\x1b[37m[17:08:15.977]\x1b[0m \x1b[91mERROR:\x1b[0m \x1b[97mcommand failed\x1b[0m \x1b[90m{\n  \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:08:03.7881995Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\\\", ActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", SubActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \\u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n}\x1b[0m\n",
    }

alert fired 4 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T17:45:42Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j0325120-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.121:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0325120/providers/microsoft.monitor/accounts/hcps-j0325120", namespace="ocm-arohcpci01-2tfb63s18q5cesobi99a7b7bb7k3a72n-j3k0c0d9n5c8h5t", pod="arohcp-monitor-kube-state-metrics-7988dc648c-6xsl5", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tfb63s18q5cesobi99a7b7bb7k3a72n-j3k0c0d9n5c8h5t/capi-provider has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-15T17:45:42Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j0325120-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cluster-api", endpoint="http", environment="ci01", instance="10.128.64.121:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0325120/providers/microsoft.monitor/accounts/hcps-j0325120", namespace="ocm-arohcpci01-2tfb63s18q5cesobi99a7b7bb7k3a72n-j3k0c0d9n5c8h5t", pod="arohcp-monitor-kube-state-metrics-7988dc648c-6xsl5", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tfb63s18q5cesobi99a7b7bb7k3a72n-j3k0c0d9n5c8h5t/cluster-api has not matched the expected number of replicas for longer than 30 minutes.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-15T17:45:42Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j0325120-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cluster-api", endpoint="http", environment="ci01", instance="10.128.64.121:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0325120/providers/microsoft.monitor/accounts/hcps-j0325120", namespace="ocm-arohcpci01-2tfb63s18q5cesobi99a7b7bb7k3a72n-j3k0c0d9n5c8h5t", pod="arohcp-monitor-kube-state-metrics-7988dc648c-6xsl5", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tfb63s18q5cesobi99a7b7bb7k3a72n-j3k0c0d9n5c8h5t/cluster-api has not matched the expected number of replicas for longer than 30 minutes.
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-15T17:45:42Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j0325120-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.121:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0325120/providers/microsoft.monitor/accounts/hcps-j0325120", namespace="ocm-arohcpci01-2tfb63s18q5cesobi99a7b7bb7k3a72n-j3k0c0d9n5c8h5t", pod="arohcp-monitor-kube-state-metrics-7988dc648c-6xsl5", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tfb63s18q5cesobi99a7b7bb7k3a72n-j3k0c0d9n5c8h5t/capi-provider has not matched the expected number of replicas for longer than 30 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 615.207s.
Expected success, but got an error:
    <*fmt.wrapError | 0x55a339ed620>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);
    let _endTime = datetime(2026-10-15T17:04:06.8555808Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-rdrv6kmh5rwg'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-15T17:04:20.6431913Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3", ActivityId:"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20", SubActivityId:"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20", ActivityType:"GW.Http.CallContext", ParentActivityId:"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20", ActivityStack:"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 > GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:04:20.6431913Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\", ActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", SubActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityStack:\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 > GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x55a33cca5a0>{
            errs: [
                <*fmt.wrapError | 0x55a339ed600>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:04:20.6431913Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\", ActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", SubActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityStack:\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 > GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x55a339ed5e0>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:04:20.6431913Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\", ActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", SubActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityStack:\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 > GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x55a331fa980>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:04:20.6431913Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\", ActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", SubActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityStack:\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 > GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x55a331fa920>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:04:20.6431913Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\", ActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", SubActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityStack:\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 > GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x55a32a9e000>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 615.207s.
Expected success, but got an error:
    <*fmt.wrapError | 0x55a339ed620>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);
    let _endTime = datetime(2026-10-15T17:04:06.8555808Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-rdrv6kmh5rwg'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-15T17:04:20.6431913Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3", ActivityId:"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20", SubActivityId:"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20", ActivityType:"GW.Http.CallContext", ParentActivityId:"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20", ActivityStack:"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 > GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:04:20.6431913Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\", ActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", SubActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityStack:\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 > GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x55a33cca5a0>{
            errs: [
                <*fmt.wrapError | 0x55a339ed600>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:04:20.6431913Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\", ActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", SubActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityStack:\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 > GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x55a339ed5e0>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:04:20.6431913Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\", ActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", SubActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityStack:\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 > GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x55a331fa980>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:04:20.6431913Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\", ActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", SubActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityStack:\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 > GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x55a331fa920>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:04:20.6431913Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\", ActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", SubActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityStack:\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 > GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x55a32a9e000>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }

time=2026-10-15T16:05:19.562Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description="Step cluster\n  Kind: ARM\n  Template: templates/svc-cluster.bicep\n  Parameters: configurations/svc-cluster.tmpl.bicepparam"
time=2026-10-15T16:05:21.120Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster
time=2026-10-15T16:05:22.261Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=ad4f22f5700ae2c1b17a10ec4808d609b2c303e3c28bbb659e87bff4bc468efa portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1280384-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fad4f22f5700ae2c1b17a10ec4808d609b2c303e3c28bbb659e87bff4bc468efa
time=2026-10-15T16:07:52.843Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1280384-svc/providers/Microsoft.Resources/deployments/ad4f22f5700ae2c1b17a10ec4808d609b2c303e3c28bbb659e87bff4bc468efa/operationStatuses/08584095265638085240\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1280384-svc/providers/Microsoft.Resources/deployments/cluster-ohrur7q7uyiys\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1280384-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j1280384-svc\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\",\\r\\n                \\\"message\\\": \\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\"https://ah-ci01-se-j1280384-1.vault.azure.net\\\\\\\"\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fkmrv5gzfs2h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x324d6bb84720>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fkmrv5gzfs2h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fkmrv5gzfs2h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T16:15:17.4151431Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fkmrv5gzfs2h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fkmrv5gzfs2h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T16:15:17.4151431Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x324d6bb84500>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fkmrv5gzfs2h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T16:15:17.4151431Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x324d6b4894f0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x324d6bb844e0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fkmrv5gzfs2h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T16:15:17.4151431Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x324d6d0c74a0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fkmrv5gzfs2h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x324d6bb84720>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fkmrv5gzfs2h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fkmrv5gzfs2h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T16:15:17.4151431Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fkmrv5gzfs2h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fkmrv5gzfs2h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T16:15:17.4151431Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x324d6bb84500>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fkmrv5gzfs2h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T16:15:17.4151431Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x324d6b4894f0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x324d6bb844e0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fkmrv5gzfs2h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T16:15:17.4151431Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x324d6d0c74a0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

time=2026-10-15T15:32:13.877Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 description="Step storageclass\n Kind: Helm\n"
time=2026-10-15T15:32:14.512Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 values="map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:d3bdf5672c862ebc0a35ea497266f8c7598862b9377e076ed14ffc329be19981]"
time=2026-10-15T15:32:14.512Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1
time=2026-10-15T15:32:14.779Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 namespace=default-sc
time=2026-10-15T15:32:15.109Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 dryRun=false
time=2026-10-15T15:32:15.109Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1
time=2026-10-15T15:32:15.240Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1
time=2026-10-15T15:33:30.310Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err="failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1"
time=2026-10-15T15:33:30.310Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err="stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1"

time=2026-10-15T14:54:42.279Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description="Step cluster\n  Kind: ARM\n  Template: templates/svc-cluster.bicep\n  Parameters: configurations/svc-cluster.tmpl.bicepparam"
time=2026-10-15T14:54:43.617Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster
time=2026-10-15T14:54:48.980Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=f4d1aa6802150b8b8948aafa95347815b90b5739192ecdc63825f2c388ca8b96 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7523328-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff4d1aa6802150b8b8948aafa95347815b90b5739192ecdc63825f2c388ca8b96
time=2026-10-15T14:57:19.554Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7523328-svc/providers/Microsoft.Resources/deployments/f4d1aa6802150b8b8948aafa95347815b90b5739192ecdc63825f2c388ca8b96/operationStatuses/08584095308013146388\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7523328-svc/providers/Microsoft.Resources/deployments/cluster-5iybzjpsz3iim\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7523328-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j7523328-svc\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\",\\r\\n                \\\"message\\\": \\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\"https://ah-ci01-se-j7523328-1.vault.azure.net\\\\\\\"\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool "np-1" for fips-enabled cluster "fips-enabled-cluster"
Unexpected error:
    <*fmt.wrapError | 0x39a645494420>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "fips-enabled-cluster" in resourcegroup="fips-enabled-tfxpkjkjrqvw" to finish creating: GET https://rp.j8228736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54644c94-7f95-4829-9f03-e9b939189ba9
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54644c94-7f95-4829-9f03-e9b939189ba9",
      "name": "54644c94-7f95-4829-9f03-e9b939189ba9",
      "status": "Failed",
      "startTime": "2026-10-15T15:18:30.534168337Z",
      "endTime": "2026-10-15T15:47:34.762349229Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine q2n5p1l1g7q3y7a-np-1-6mdqn-p62bb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-tfxpkjkjrqvw--managed/providers/Microsoft.Compute/virtualMachines/q2n5p1l1g7q3y7a-np-1-6mdqn-p62bb to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine q2n5p1l1g7q3y7a-np-1-6mdqn-p62bb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-tfxpkjkjrqvw--managed/providers/Microsoft.Compute/virtualMachines/q2n5p1l1g7q3y7a-np-1-6mdqn-p62bb to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8228736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54644c94-7f95-4829-9f03-e9b939189ba9
            correlation ID: af51592b-1bd9-4ddd-81a5-d7c702c77eb4
            client request ID: e54f4800-8170-46d4-a0fb-e016b4fc8317
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool "np-1" for fips-enabled cluster "fips-enabled-cluster"
Unexpected error:
    <*fmt.wrapError | 0x39a645494420>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "fips-enabled-cluster" in resourcegroup="fips-enabled-tfxpkjkjrqvw" to finish creating: GET https://rp.j8228736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54644c94-7f95-4829-9f03-e9b939189ba9
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54644c94-7f95-4829-9f03-e9b939189ba9",
      "name": "54644c94-7f95-4829-9f03-e9b939189ba9",
      "status": "Failed",
      "startTime": "2026-10-15T15:18:30.534168337Z",
      "endTime": "2026-10-15T15:47:34.762349229Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine q2n5p1l1g7q3y7a-np-1-6mdqn-p62bb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-tfxpkjkjrqvw--managed/providers/Microsoft.Compute/virtualMachines/q2n5p1l1g7q3y7a-np-1-6mdqn-p62bb to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine q2n5p1l1g7q3y7a-np-1-6mdqn-p62bb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-tfxpkjkjrqvw--managed/providers/Microsoft.Compute/virtualMachines/q2n5p1l1g7q3y7a-np-1-6mdqn-p62bb to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8228736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54644c94-7f95-4829-9f03-e9b939189ba9
            correlation ID: af51592b-1bd9-4ddd-81a5-d7c702c77eb4
            client request ID: e54f4800-8170-46d4-a0fb-e016b4fc8317
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np
Unexpected error:
    <*fmt.wrapError | 0x11568de31de0>: 
    failed to create NodePool cordon-np: failed waiting for nodepool="cordon-np" for cluster "cordon-hcp-cluster" in resourcegroup="rg-cordon-test-c7lxc6fklr7v" to finish creating: GET https://rp.j3811840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e291eb94-ff6c-44e3-80a6-4f67644a7cda
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e291eb94-ff6c-44e3-80a6-4f67644a7cda",
      "name": "e291eb94-ff6c-44e3-80a6-4f67644a7cda",
      "status": "Failed",
      "startTime": "2026-10-15T15:17:13.641479816Z",
      "endTime": "2026-10-15T15:36:13.431611364Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine v5s2v4q6s4c1e9i-cordon-np-7nlxl-4n292: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-c7lxc6fklr7v--managed/providers/Microsoft.Compute/virtualMachines/v5s2v4q6s4c1e9i-cordon-np-7nlxl-4n292 to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine v5s2v4q6s4c1e9i-cordon-np-7nlxl-4n292: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-c7lxc6fklr7v--managed/providers/Microsoft.Compute/virtualMachines/v5s2v4q6s4c1e9i-cordon-np-7nlxl-4n292 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3811840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e291eb94-ff6c-44e3-80a6-4f67644a7cda
            correlation ID: e979f666-45d1-4a7d-8462-7e9f430e72de
            client request ID: 24ecc636-3ded-42e9-ad02-a9811b79b2e6
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np
Unexpected error:
    <*fmt.wrapError | 0x11568de31de0>: 
    failed to create NodePool cordon-np: failed waiting for nodepool="cordon-np" for cluster "cordon-hcp-cluster" in resourcegroup="rg-cordon-test-c7lxc6fklr7v" to finish creating: GET https://rp.j3811840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e291eb94-ff6c-44e3-80a6-4f67644a7cda
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e291eb94-ff6c-44e3-80a6-4f67644a7cda",
      "name": "e291eb94-ff6c-44e3-80a6-4f67644a7cda",
      "status": "Failed",
      "startTime": "2026-10-15T15:17:13.641479816Z",
      "endTime": "2026-10-15T15:36:13.431611364Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine v5s2v4q6s4c1e9i-cordon-np-7nlxl-4n292: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-c7lxc6fklr7v--managed/providers/Microsoft.Compute/virtualMachines/v5s2v4q6s4c1e9i-cordon-np-7nlxl-4n292 to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine v5s2v4q6s4c1e9i-cordon-np-7nlxl-4n292: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-c7lxc6fklr7v--managed/providers/Microsoft.Compute/virtualMachines/v5s2v4q6s4c1e9i-cordon-np-7nlxl-4n292 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3811840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e291eb94-ff6c-44e3-80a6-4f67644a7cda
            correlation ID: e979f666-45d1-4a7d-8462-7e9f430e72de
            client request ID: 24ecc636-3ded-42e9-ad02-a9811b79b2e6
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool "np-1" for fips-enabled cluster "fips-enabled-cluster"
Unexpected error:
    <*fmt.wrapError | 0x39d2b3425680>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "fips-enabled-cluster" in resourcegroup="fips-enabled-8qwb28xp7sbt" to finish creating: GET https://rp.j6067584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5e84a58-f5a8-4651-bb4c-046ebf84406a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5e84a58-f5a8-4651-bb4c-046ebf84406a",
      "name": "c5e84a58-f5a8-4651-bb4c-046ebf84406a",
      "status": "Failed",
      "startTime": "2026-10-15T15:12:44.954501704Z",
      "endTime": "2026-10-15T15:41:51.289310045Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine l5l1v8m6s1e4e4u-np-1-29llz-nclmc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-8qwb28xp7sbt--managed/providers/Microsoft.Compute/virtualMachines/l5l1v8m6s1e4e4u-np-1-29llz-nclmc to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine l5l1v8m6s1e4e4u-np-1-29llz-nclmc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-8qwb28xp7sbt--managed/providers/Microsoft.Compute/virtualMachines/l5l1v8m6s1e4e4u-np-1-29llz-nclmc to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6067584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5e84a58-f5a8-4651-bb4c-046ebf84406a
            correlation ID: 37d1dd03-a5b0-483e-bcae-0de26622ad50
            client request ID: 56144e42-9db3-4257-909a-455e8e0e068f
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool "np-1" for fips-enabled cluster "fips-enabled-cluster"
Unexpected error:
    <*fmt.wrapError | 0x39d2b3425680>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "fips-enabled-cluster" in resourcegroup="fips-enabled-8qwb28xp7sbt" to finish creating: GET https://rp.j6067584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5e84a58-f5a8-4651-bb4c-046ebf84406a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5e84a58-f5a8-4651-bb4c-046ebf84406a",
      "name": "c5e84a58-f5a8-4651-bb4c-046ebf84406a",
      "status": "Failed",
      "startTime": "2026-10-15T15:12:44.954501704Z",
      "endTime": "2026-10-15T15:41:51.289310045Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine l5l1v8m6s1e4e4u-np-1-29llz-nclmc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-8qwb28xp7sbt--managed/providers/Microsoft.Compute/virtualMachines/l5l1v8m6s1e4e4u-np-1-29llz-nclmc to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine l5l1v8m6s1e4e4u-np-1-29llz-nclmc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-8qwb28xp7sbt--managed/providers/Microsoft.Compute/virtualMachines/l5l1v8m6s1e4e4u-np-1-29llz-nclmc to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6067584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5e84a58-f5a8-4651-bb4c-046ebf84406a
            correlation ID: 37d1dd03-a5b0-483e-bcae-0de26622ad50
            client request ID: 56144e42-9db3-4257-909a-455e8e0e068f
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 610.563s.
Expected success, but got an error:
    <*fmt.wrapError | 0x37a644e389e0>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);
    let _endTime = datetime(2026-10-15T15:31:15.7400199Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-qsncztkggf76'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-15T15:31:33.5544050Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49", ActivityId:"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8", SubActivityId:"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8", ActivityType:"GW.Http.CallContext", ParentActivityId:"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8", ActivityStack:"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 > GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qsncztkggf76'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T15:31:33.5544050Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\", ActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", SubActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 > GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x37a64583fda0>{
            errs: [
                <*fmt.wrapError | 0x37a644e389c0>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qsncztkggf76'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T15:31:33.5544050Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\", ActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", SubActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 > GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x37a644e389a0>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qsncztkggf76'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T15:31:33.5544050Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\", ActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", SubActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 > GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x37a645c4b2c0>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qsncztkggf76'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T15:31:33.5544050Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\", ActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", SubActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 > GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x37a645c4b260>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qsncztkggf76'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T15:31:33.5544050Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\", ActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", SubActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 > GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x37a646042000>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 610.563s.
Expected success, but got an error:
    <*fmt.wrapError | 0x37a644e389e0>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);
    let _endTime = datetime(2026-10-15T15:31:15.7400199Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-qsncztkggf76'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-15T15:31:33.5544050Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49", ActivityId:"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8", SubActivityId:"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8", ActivityType:"GW.Http.CallContext", ParentActivityId:"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8", ActivityStack:"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 > GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qsncztkggf76'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T15:31:33.5544050Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\", ActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", SubActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 > GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x37a64583fda0>{
            errs: [
                <*fmt.wrapError | 0x37a644e389c0>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qsncztkggf76'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T15:31:33.5544050Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\", ActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", SubActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 > GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x37a644e389a0>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qsncztkggf76'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T15:31:33.5544050Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\", ActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", SubActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 > GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x37a645c4b2c0>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qsncztkggf76'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T15:31:33.5544050Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\", ActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", SubActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 > GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x37a645c4b260>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-qsncztkggf76'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T15:31:33.5544050Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\", ActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", SubActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 > GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x37a646042000>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }

alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T16:05:17Z
  Severity:    Sev3
  Labels:      alertname="LeaderElectionLeaseStale", cluster="ci01-j6067584-mgmt-2", component="leader-election", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", lease="work-manager", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6067584/providers/microsoft.monitor/accounts/services-j6067584", namespace="klusterlet-2tf9plfjqn7ti2g60b9k20vuirc4k7vu", pod="arohcp-monitor-kube-state-metrics-7988dc648c-g6pj5", prometheus="prometheus/prometheus", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Leader election lease work-manager in namespace klusterlet-2tf9plfjqn7ti2g60b9k20vuirc4k7vu on cluster ci01-j6067584-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-kzwmzc2svmh2/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-kzwmzc2svmh2/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed

fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:116]: Cluster GET returned error: Get "https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-credential-lifecycle-test-zzgk4tx5xdlx?api-version=2021-04-01": stream error: stream ID 53; CANCEL; received from peer
fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:116]: Cluster GET returned error: Get "https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-credential-lifecycle-test-zzgk4tx5xdlx?api-version=2021-04-01": stream error: stream ID 53; CANCEL; received from peer

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool "np-1" for fips-enabled cluster "fips-enabled-cluster"
Unexpected error:
    <*fmt.wrapError | 0x5786d893780>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "fips-enabled-cluster" in resourcegroup="fips-enabled-p5l925vxtbf2" to finish creating: GET https://rp.j4585728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e4240caf-faff-4dd3-8518-154e6ce0cf5f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e4240caf-faff-4dd3-8518-154e6ce0cf5f",
      "name": "e4240caf-faff-4dd3-8518-154e6ce0cf5f",
      "status": "Failed",
      "startTime": "2026-10-15T14:50:26.445767969Z",
      "endTime": "2026-10-15T15:19:34.559056954Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine c2q3j3s6u4f5i0i-np-1-sgjbn-px8rw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-p5l925vxtbf2--managed/providers/Microsoft.Compute/virtualMachines/c2q3j3s6u4f5i0i-np-1-sgjbn-px8rw to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine c2q3j3s6u4f5i0i-np-1-sgjbn-px8rw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-p5l925vxtbf2--managed/providers/Microsoft.Compute/virtualMachines/c2q3j3s6u4f5i0i-np-1-sgjbn-px8rw to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4585728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e4240caf-faff-4dd3-8518-154e6ce0cf5f
            correlation ID: 4cec60dd-ea89-4f5c-ac3c-ea60f8df07f2
            client request ID: f0d2dea7-1bed-428e-b831-0a9033f9f104
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool "np-1" for fips-enabled cluster "fips-enabled-cluster"
Unexpected error:
    <*fmt.wrapError | 0x5786d893780>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "fips-enabled-cluster" in resourcegroup="fips-enabled-p5l925vxtbf2" to finish creating: GET https://rp.j4585728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e4240caf-faff-4dd3-8518-154e6ce0cf5f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e4240caf-faff-4dd3-8518-154e6ce0cf5f",
      "name": "e4240caf-faff-4dd3-8518-154e6ce0cf5f",
      "status": "Failed",
      "startTime": "2026-10-15T14:50:26.445767969Z",
      "endTime": "2026-10-15T15:19:34.559056954Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine c2q3j3s6u4f5i0i-np-1-sgjbn-px8rw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-p5l925vxtbf2--managed/providers/Microsoft.Compute/virtualMachines/c2q3j3s6u4f5i0i-np-1-sgjbn-px8rw to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine c2q3j3s6u4f5i0i-np-1-sgjbn-px8rw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-p5l925vxtbf2--managed/providers/Microsoft.Compute/virtualMachines/c2q3j3s6u4f5i0i-np-1-sgjbn-px8rw to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4585728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e4240caf-faff-4dd3-8518-154e6ce0cf5f
            correlation ID: 4cec60dd-ea89-4f5c-ac3c-ea60f8df07f2
            client request ID: f0d2dea7-1bed-428e-b831-0a9033f9f104
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:365]: Timed out after 300.000s.
ClusterIssuer should become Ready (ACME account registered)
Expected success, but got an error:
    <*errors.errorString | 0x2b9d750c1520>: 
    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get "https://acme-staging-v02.api.letsencrypt.org/directory": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58152->172.30.0.10:53: read: connection refused
    {
        s: "clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58152->172.30.0.10:53: read: connection refused",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:365]: Timed out after 300.000s.
ClusterIssuer should become Ready (ACME account registered)
Expected success, but got an error:
    <*errors.errorString | 0x2b9d750c1520>: 
    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get "https://acme-staging-v02.api.letsencrypt.org/directory": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58152->172.30.0.10:53: read: connection refused
    {
        s: "clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58152->172.30.0.10:53: read: connection refused",
    }

time=2026-10-15T14:28:12.191Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image description="Step mirror-image\n  Kind: ImageMirror\n  From {<nil> clustersService.image.registry <nil>}:{<nil> clustersService.image.repository <nil>}@{<nil> clustersService.image.digest <nil>} to {<nil> acr.svc.name <nil>}\n"
time=2026-10-15T14:28:45.415Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image err="error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\nFetch pull secret for source registry arohcpsvcdev from arohcpdev-global KV.\nLogging into target ACR arohcpsvcdev.\nLogin Succeeded\nMirroring image arohcpsvcdev/nshneor/cs@sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d to arohcpsvcdev.azurecr.io/nshneor/cs:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d.\nThe image will still be available under it's original digest sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d in the target registry.\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\nCommand failed (attempt 1/5). Retrying in 2s...\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\nCommand failed (attempt 2/5). Retrying in 4s...\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\nCommand failed (attempt 3/5). Retrying in 8s...\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\nCommand failed (attempt 4/5). Retrying in 16s...\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\nCommand failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d arohcpsvcdev.azurecr.io/nshneor/cs:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d --from-registry-config /tmp/tmp.X2puJ9ooST/containers/auth.json --to-registry-config /tmp/tmp.X2puJ9ooST/containers/auth.json\n exit status 1"

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54bm2656zv2m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1241fd1c760>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54bm2656zv2m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54bm2656zv2m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T14:39:07.1748687Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54bm2656zv2m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54bm2656zv2m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T14:39:07.1748687Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1241fd1c740>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54bm2656zv2m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T14:39:07.1748687Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1241ee2e870>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1241fd1c720>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54bm2656zv2m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T14:39:07.1748687Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x12421019b00>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54bm2656zv2m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1241fd1c760>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54bm2656zv2m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54bm2656zv2m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T14:39:07.1748687Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54bm2656zv2m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54bm2656zv2m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T14:39:07.1748687Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1241fd1c740>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54bm2656zv2m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T14:39:07.1748687Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1241ee2e870>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1241fd1c720>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54bm2656zv2m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T14:39:07.1748687Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x12421019b00>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

alert fired 21 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:53Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-bbr5b", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-bbr5b has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:55Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-mcvcl", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-mcvcl has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:54Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-fjcvt", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-fjcvt has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:55Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-slcz2", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-slcz2 has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:54Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-bb5c4", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-bb5c4 has been in a non-ready state for longer than 5 minutes.
Firing 6:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:54Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-6wb8g", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-6wb8g has been in a non-ready state for longer than 5 minutes.
Firing 7:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:54Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-dvq7x", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-dvq7x has been in a non-ready state for longer than 5 minutes.
Firing 8:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:57Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-4c7nd", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-4c7nd has been in a non-ready state for longer than 5 minutes.
Firing 9:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:55Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-nf785", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-nf785 has been in a non-ready state for longer than 5 minutes.
Firing 10:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:54Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-bj49s", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-bj49s has been in a non-ready state for longer than 5 minutes.
Firing 11:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-l6wwg", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-l6wwg has been in a non-ready state for longer than 5 minutes.
Firing 12:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:57Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-65c77", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-65c77 has been in a non-ready state for longer than 5 minutes.
Firing 13:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-7j5wq", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-7j5wq has been in a non-ready state for longer than 5 minutes.
Firing 14:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:55Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-dbzxn", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-dbzxn has been in a non-ready state for longer than 5 minutes.
Firing 15:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-bb4h2", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-bb4h2 has been in a non-ready state for longer than 5 minutes.
Firing 16:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-bphhj", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-bphhj has been in a non-ready state for longer than 5 minutes.
Firing 17:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-wvg8r", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-wvg8r has been in a non-ready state for longer than 5 minutes.
Firing 18:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-kwr9x", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-kwr9x has been in a non-ready state for longer than 5 minutes.
Firing 19:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:54Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-p7zfh", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-p7zfh has been in a non-ready state for longer than 5 minutes.
Firing 20:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:55Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-pjpzq", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-pjpzq has been in a non-ready state for longer than 5 minutes.
Firing 21:
  State:       Resolved
  Started:     2026-10-15T14:57:52Z
  Ended:       2026-10-15T15:13:55Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2098432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas", pod="kube-controller-manager-5b56bb49b9-xtqlx", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tf9fdt61q7jle8eqdo7ou1ubc913lca-private-kas/kube-controller-manager-5b56bb49b9-xtqlx has been in a non-ready state for longer than 5 minutes.

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T14:41:42Z
  Ended:       2026-10-15T15:02:43Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j1944960-svc", component="backend", name="clusterresources", region="centralus", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
Firing 2:
  State:       Resolved
  Started:     2026-10-15T14:41:42Z
  Ended:       2026-10-15T15:03:42Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j1944960-svc", component="backend", name="csstatedump", region="centralus", severity="warning"
  Description: Backend controller workqueue csstatedump has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:
    <*errors.joinError | 0xa89f6d94528>: 
    resource group "rg-np-noedge-tw7cjj-xlndscvj7dwp": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="np-noedge-tw7cjj" in resourcegroup="rg-np-noedge-tw7cjj-xlndscvj7dwp" to finish deleting: GET https://rp.j5616384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/19704973-55a6-47ab-b561-11bb7cb66fb8
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/19704973-55a6-47ab-b561-11bb7cb66fb8",
      "name": "19704973-55a6-47ab-b561-11bb7cb66fb8",
      "status": "Failed",
      "startTime": "2026-10-15T14:51:30.063016611Z",
      "endTime": "2026-10-15T15:15:36.942553753Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tf9cp7cug191vvs2krpcsqb89jucjnj still exists (deletion dispatched at 2026-10-15T14:51:30Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5616384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/19704973-55a6-47ab-b561-11bb7cb66fb8
            correlation ID: 6de0dbef-67fc-48b2-ad3a-a51b2af3faa1
            client request ID: a410cfe8-06c4-4eb7-afde-a18a2ead7f72
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:
    <*errors.joinError | 0xa89f6d94528>: 
    resource group "rg-np-noedge-tw7cjj-xlndscvj7dwp": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="np-noedge-tw7cjj" in resourcegroup="rg-np-noedge-tw7cjj-xlndscvj7dwp" to finish deleting: GET https://rp.j5616384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/19704973-55a6-47ab-b561-11bb7cb66fb8
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/19704973-55a6-47ab-b561-11bb7cb66fb8",
      "name": "19704973-55a6-47ab-b561-11bb7cb66fb8",
      "status": "Failed",
      "startTime": "2026-10-15T14:51:30.063016611Z",
      "endTime": "2026-10-15T15:15:36.942553753Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tf9cp7cug191vvs2krpcsqb89jucjnj still exists (deletion dispatched at 2026-10-15T14:51:30Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5616384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/19704973-55a6-47ab-b561-11bb7cb66fb8
            correlation ID: 6de0dbef-67fc-48b2-ad3a-a51b2af3faa1
            client request ID: a410cfe8-06c4-4eb7-afde-a18a2ead7f72
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool "np-1" for v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0x30ba0d996320>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "v20261001" in resourcegroup="v20261001-kv7nqskz8rbt" to finish creating: GET https://rp.j0712192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf68877b-df8e-430d-ab1f-aa5c781ac5c2
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf68877b-df8e-430d-ab1f-aa5c781ac5c2",
      "name": "bf68877b-df8e-430d-ab1f-aa5c781ac5c2",
      "status": "Failed",
      "startTime": "2026-10-15T14:23:20.152433644Z",
      "endTime": "2026-10-15T14:52:22.758892045Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine v20261001-np-1-7zsxm-lwj52: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-kv7nqskz8rbt--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-7zsxm-lwj52 to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine v20261001-np-1-7zsxm-lwj52: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-kv7nqskz8rbt--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-7zsxm-lwj52 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0712192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf68877b-df8e-430d-ab1f-aa5c781ac5c2
            correlation ID: 6f1d82bf-560b-4ab8-8830-31f9f4f34d95
            client request ID: f649116d-c373-4f92-866d-87f54a1d5dfb
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool "np-1" for v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0x30ba0d996320>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "v20261001" in resourcegroup="v20261001-kv7nqskz8rbt" to finish creating: GET https://rp.j0712192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf68877b-df8e-430d-ab1f-aa5c781ac5c2
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf68877b-df8e-430d-ab1f-aa5c781ac5c2",
      "name": "bf68877b-df8e-430d-ab1f-aa5c781ac5c2",
      "status": "Failed",
      "startTime": "2026-10-15T14:23:20.152433644Z",
      "endTime": "2026-10-15T14:52:22.758892045Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine v20261001-np-1-7zsxm-lwj52: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-kv7nqskz8rbt--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-7zsxm-lwj52 to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine v20261001-np-1-7zsxm-lwj52: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-kv7nqskz8rbt--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-7zsxm-lwj52 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0712192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf68877b-df8e-430d-ab1f-aa5c781ac5c2
            correlation ID: 6f1d82bf-560b-4ab8-8830-31f9f4f34d95
            client request ID: f649116d-c373-4f92-866d-87f54a1d5dfb
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fvsjnwnr4hjp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x22b4f5a38860>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fvsjnwnr4hjp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fvsjnwnr4hjp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T14:10:13.1708787Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fvsjnwnr4hjp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fvsjnwnr4hjp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T14:10:13.1708787Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x22b4f5a38840>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fvsjnwnr4hjp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T14:10:13.1708787Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x22b4f55e6780>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x22b4f5a38820>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fvsjnwnr4hjp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T14:10:13.1708787Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x22b4f5d16570>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fvsjnwnr4hjp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x22b4f5a38860>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fvsjnwnr4hjp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fvsjnwnr4hjp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T14:10:13.1708787Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fvsjnwnr4hjp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fvsjnwnr4hjp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T14:10:13.1708787Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x22b4f5a38840>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fvsjnwnr4hjp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T14:10:13.1708787Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x22b4f55e6780>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x22b4f5a38820>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fvsjnwnr4hjp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T14:10:13.1708787Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x22b4f5d16570>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np
Unexpected error:
    <*fmt.wrapError | 0x359ec43a40a0>: 
    failed to create NodePool cordon-np: failed waiting for nodepool="cordon-np" for cluster "cordon-hcp-cluster" in resourcegroup="rg-cordon-test-dn8lfjb6zdxg" to finish creating: GET https://rp.j0994688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4720c16-11a5-4346-98ae-75d3b08058cb
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4720c16-11a5-4346-98ae-75d3b08058cb",
      "name": "a4720c16-11a5-4346-98ae-75d3b08058cb",
      "status": "Failed",
      "startTime": "2026-10-15T14:07:17.645930672Z",
      "endTime": "2026-10-15T14:36:19.507334122Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine f4r7v6u6v4w8n2i-cordon-np-lb7sg-4n2l5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-dn8lfjb6zdxg--managed/providers/Microsoft.Compute/virtualMachines/f4r7v6u6v4w8n2i-cordon-np-lb7sg-4n2l5 to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine f4r7v6u6v4w8n2i-cordon-np-lb7sg-4n2l5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-dn8lfjb6zdxg--managed/providers/Microsoft.Compute/virtualMachines/f4r7v6u6v4w8n2i-cordon-np-lb7sg-4n2l5 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0994688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4720c16-11a5-4346-98ae-75d3b08058cb
            correlation ID: e54bc3dc-a2f8-4ddc-8bc6-340483245a5f
            client request ID: 5228ec88-3157-4545-ad9a-3babd5328eb6
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np
Unexpected error:
    <*fmt.wrapError | 0x359ec43a40a0>: 
    failed to create NodePool cordon-np: failed waiting for nodepool="cordon-np" for cluster "cordon-hcp-cluster" in resourcegroup="rg-cordon-test-dn8lfjb6zdxg" to finish creating: GET https://rp.j0994688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4720c16-11a5-4346-98ae-75d3b08058cb
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4720c16-11a5-4346-98ae-75d3b08058cb",
      "name": "a4720c16-11a5-4346-98ae-75d3b08058cb",
      "status": "Failed",
      "startTime": "2026-10-15T14:07:17.645930672Z",
      "endTime": "2026-10-15T14:36:19.507334122Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine f4r7v6u6v4w8n2i-cordon-np-lb7sg-4n2l5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-dn8lfjb6zdxg--managed/providers/Microsoft.Compute/virtualMachines/f4r7v6u6v4w8n2i-cordon-np-lb7sg-4n2l5 to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine f4r7v6u6v4w8n2i-cordon-np-lb7sg-4n2l5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-dn8lfjb6zdxg--managed/providers/Microsoft.Compute/virtualMachines/f4r7v6u6v4w8n2i-cordon-np-lb7sg-4n2l5 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0994688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4720c16-11a5-4346-98ae-75d3b08058cb
            correlation ID: e54bc3dc-a2f8-4ddc-8bc6-340483245a5f
            client request ID: 5228ec88-3157-4545-ad9a-3babd5328eb6
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.
Expected success, but got an error:
    <*fmt.wrapError | 0x11bd47e7c4a0>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);
    let _endTime = datetime(2026-10-15T14:26:17.9586903Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-hzvnhdkdhrhv'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-15T14:26:30.5366490Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10", ActivityId:"53ab61d9-b679-46ee-bc5b-6292ddb0a06e", SubActivityId:"53ab61d9-b679-46ee-bc5b-6292ddb0a06e", ActivityType:"GW.Http.CallContext", ParentActivityId:"53ab61d9-b679-46ee-bc5b-6292ddb0a06e", ActivityStack:"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e > GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T14:26:30.5366490Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\", ActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", SubActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e > GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x11bd532fd080>{
            errs: [
                <*fmt.wrapError | 0x11bd47e7c480>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T14:26:30.5366490Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\", ActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", SubActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e > GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x11bd47e7c440>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T14:26:30.5366490Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\", ActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", SubActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e > GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x11bd49601a80>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T14:26:30.5366490Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\", ActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", SubActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e > GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x11bd49601a20>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T14:26:30.5366490Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\", ActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", SubActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e > GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x11bd4901a900>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.
Expected success, but got an error:
    <*fmt.wrapError | 0x11bd47e7c4a0>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);
    let _endTime = datetime(2026-10-15T14:26:17.9586903Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-hzvnhdkdhrhv'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-15T14:26:30.5366490Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10", ActivityId:"53ab61d9-b679-46ee-bc5b-6292ddb0a06e", SubActivityId:"53ab61d9-b679-46ee-bc5b-6292ddb0a06e", ActivityType:"GW.Http.CallContext", ParentActivityId:"53ab61d9-b679-46ee-bc5b-6292ddb0a06e", ActivityStack:"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e > GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T14:26:30.5366490Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\", ActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", SubActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e > GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x11bd532fd080>{
            errs: [
                <*fmt.wrapError | 0x11bd47e7c480>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T14:26:30.5366490Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\", ActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", SubActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e > GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x11bd47e7c440>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T14:26:30.5366490Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\", ActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", SubActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e > GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x11bd49601a80>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T14:26:30.5366490Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\", ActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", SubActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e > GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x11bd49601a20>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T14:26:30.5366490Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\", ActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", SubActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e > GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x11bd4901a900>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T14:15:22Z
  Ended:       2026-10-15T14:36:22Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j4293120-svc", component="backend", name="actualhostedcluster", region="centralus", severity="warning"
  Description: Backend controller workqueue actualhostedcluster has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

time=2026-10-15T13:13:09.548Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description="Step cluster\n  Kind: ARM\n  Template: templates/svc-cluster.bicep\n  Parameters: configurations/svc-cluster.tmpl.bicepparam"
time=2026-10-15T13:13:11.116Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster
time=2026-10-15T13:13:12.381Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=c62031d0e64aeaceb5347a6a10975782ee41b3777d31a7e45f560a99656b0f86 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7456896-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc62031d0e64aeaceb5347a6a10975782ee41b3777d31a7e45f560a99656b0f86
time=2026-10-15T13:15:43.061Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7456896-svc/providers/Microsoft.Resources/deployments/c62031d0e64aeaceb5347a6a10975782ee41b3777d31a7e45f560a99656b0f86/operationStatuses/08584095368938337441\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7456896-svc/providers/Microsoft.Resources/deployments/cluster-mkmoz5ke6r4xm\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7456896-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j7456896-svc\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\",\\r\\n                \\\"message\\\": \\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\"https://ah-ci01-se-j7456896-1.vault.azure.net\\\\\\\"\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

alert fired 42 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:36:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-disk-csi-driver-controller-6759fd4676-8xwhg", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-disk-csi-driver-controller-6759fd4676-8xwhg has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:36:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-b847b98c9-7m695", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-b847b98c9-7m695 has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:36:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-b847b98c9-dq7jk", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-b847b98c9-dq7jk has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:40:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-b847b98c9-k8kmv", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-b847b98c9-k8kmv has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:35:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-disk-csi-driver-controller-77857b488-g8tj5", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-disk-csi-driver-controller-77857b488-g8tj5 has been in a non-ready state for longer than 5 minutes.
Firing 6:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:35:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-disk-csi-driver-controller-77857b488-mx45b", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-disk-csi-driver-controller-77857b488-mx45b has been in a non-ready state for longer than 5 minutes.
Firing 7:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:35:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-b847b98c9-v89wh", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-b847b98c9-v89wh has been in a non-ready state for longer than 5 minutes.
Firing 8:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:39:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-disk-csi-driver-controller-77857b488-gbhw8", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-disk-csi-driver-controller-77857b488-gbhw8 has been in a non-ready state for longer than 5 minutes.
Firing 9:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:42:00Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7v6diupt5grtmeo5qd4ocn7bfci0v-d8n3d6k7c7h0h4f", pod="csi-snapshot-controller-f45b86c9d-r6g6v", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7v6diupt5grtmeo5qd4ocn7bfci0v-d8n3d6k7c7h0h4f/csi-snapshot-controller-f45b86c9d-r6g6v has been in a non-ready state for longer than 5 minutes.
Firing 10:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:35:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-b847b98c9-rcg8p", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-b847b98c9-rcg8p has been in a non-ready state for longer than 5 minutes.
Firing 11:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:36:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-disk-csi-driver-controller-77857b488-5z56c", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-disk-csi-driver-controller-77857b488-5z56c has been in a non-ready state for longer than 5 minutes.
Firing 12:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:35:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-disk-csi-driver-controller-77857b488-2k9hp", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-disk-csi-driver-controller-77857b488-2k9hp has been in a non-ready state for longer than 5 minutes.
Firing 13:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:40:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-disk-csi-driver-controller-77857b488-ljp7c", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-disk-csi-driver-controller-77857b488-ljp7c has been in a non-ready state for longer than 5 minutes.
Firing 14:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:35:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-b847b98c9-r4qdx", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-b847b98c9-r4qdx has been in a non-ready state for longer than 5 minutes.
Firing 15:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:35:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-b847b98c9-qb97t", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-b847b98c9-qb97t has been in a non-ready state for longer than 5 minutes.
Firing 16:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:36:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-disk-csi-driver-controller-77857b488-l7dm8", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-disk-csi-driver-controller-77857b488-l7dm8 has been in a non-ready state for longer than 5 minutes.
Firing 17:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:37:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7v6diupt5grtmeo5qd4ocn7bfci0v-d8n3d6k7c7h0h4f", pod="csi-snapshot-controller-f45b86c9d-z5cnv", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7v6diupt5grtmeo5qd4ocn7bfci0v-d8n3d6k7c7h0h4f/csi-snapshot-controller-f45b86c9d-z5cnv has been in a non-ready state for longer than 5 minutes.
Firing 18:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:35:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-b847b98c9-v2cjt", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-b847b98c9-v2cjt has been in a non-ready state for longer than 5 minutes.
Firing 19:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:36:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-disk-csi-driver-controller-77857b488-8pf8t", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-disk-csi-driver-controller-77857b488-8pf8t has been in a non-ready state for longer than 5 minutes.
Firing 20:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:35:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-6d749dbd59-lrz4k", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-6d749dbd59-lrz4k has been in a non-ready state for longer than 5 minutes.
Firing 21:
  State:       Resolved
  Started:     2026-10-15T13:13:59Z
  Ended:       2026-10-15T13:35:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-disk-csi-driver-controller-77857b488-lsdj2", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-disk-csi-driver-controller-77857b488-lsdj2 has been in a non-ready state for longer than 5 minutes.
Firing 22:
  State:       Resolved
  Started:     2026-10-15T13:14:00Z
  Ended:       2026-10-15T13:36:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-b847b98c9-nvjhr", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-b847b98c9-nvjhr has been in a non-ready state for longer than 5 minutes.
Firing 23:
  State:       Resolved
  Started:     2026-10-15T13:14:00Z
  Ended:       2026-10-15T13:35:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-disk-csi-driver-controller-77857b488-7k98b", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-disk-csi-driver-controller-77857b488-7k98b has been in a non-ready state for longer than 5 minutes.
Firing 24:
  State:       Resolved
  Started:     2026-10-15T13:14:00Z
  Ended:       2026-10-15T13:35:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-disk-csi-driver-controller-77857b488-dn9rl", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-disk-csi-driver-controller-77857b488-dn9rl has been in a non-ready state for longer than 5 minutes.
Firing 25:
  State:       Resolved
  Started:     2026-10-15T13:14:00Z
  Ended:       2026-10-15T13:37:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7v6diupt5grtmeo5qd4ocn7bfci0v-d8n3d6k7c7h0h4f", pod="csi-snapshot-controller-f45b86c9d-5fp9j", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7v6diupt5grtmeo5qd4ocn7bfci0v-d8n3d6k7c7h0h4f/csi-snapshot-controller-f45b86c9d-5fp9j has been in a non-ready state for longer than 5 minutes.
Firing 26:
  State:       Resolved
  Started:     2026-10-15T13:14:00Z
  Ended:       2026-10-15T13:35:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-b847b98c9-nkjlz", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-b847b98c9-nkjlz has been in a non-ready state for longer than 5 minutes.
Firing 27:
  State:       Resolved
  Started:     2026-10-15T13:14:00Z
  Ended:       2026-10-15T13:37:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7v6diupt5grtmeo5qd4ocn7bfci0v-d8n3d6k7c7h0h4f", pod="csi-snapshot-controller-f45b86c9d-96ztf", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7v6diupt5grtmeo5qd4ocn7bfci0v-d8n3d6k7c7h0h4f/csi-snapshot-controller-f45b86c9d-96ztf has been in a non-ready state for longer than 5 minutes.
Firing 28:
  State:       Resolved
  Started:     2026-10-15T13:14:00Z
  Ended:       2026-10-15T13:40:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-disk-csi-driver-controller-6759fd4676-nddr9", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-disk-csi-driver-controller-6759fd4676-nddr9 has been in a non-ready state for longer than 5 minutes.
Firing 29:
  State:       Resolved
  Started:     2026-10-15T13:14:00Z
  Ended:       2026-10-15T13:36:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-b847b98c9-bktn7", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-b847b98c9-bktn7 has been in a non-ready state for longer than 5 minutes.
Firing 30:
  State:       Resolved
  Started:     2026-10-15T13:14:00Z
  Ended:       2026-10-15T13:36:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-b847b98c9-qmtqs", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-b847b98c9-qmtqs has been in a non-ready state for longer than 5 minutes.
Firing 31:
  State:       Resolved
  Started:     2026-10-15T13:14:00Z
  Ended:       2026-10-15T13:36:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-6d749dbd59-4cjnm", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-6d749dbd59-4cjnm has been in a non-ready state for longer than 5 minutes.
Firing 32:
  State:       Resolved
  Started:     2026-10-15T13:14:00Z
  Ended:       2026-10-15T13:35:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-b847b98c9-5s5zm", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-b847b98c9-5s5zm has been in a non-ready state for longer than 5 minutes.
Firing 33:
  State:       Resolved
  Started:     2026-10-15T13:14:00Z
  Ended:       2026-10-15T13:35:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-b847b98c9-jgd2h", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-b847b98c9-jgd2h has been in a non-ready state for longer than 5 minutes.
Firing 34:
  State:       Resolved
  Started:     2026-10-15T13:14:00Z
  Ended:       2026-10-15T13:36:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-disk-csi-driver-controller-77857b488-vpsjp", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-disk-csi-driver-controller-77857b488-vpsjp has been in a non-ready state for longer than 5 minutes.
Firing 35:
  State:       Resolved
  Started:     2026-10-15T13:14:00Z
  Ended:       2026-10-15T13:35:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-b847b98c9-7w2w9", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-b847b98c9-7w2w9 has been in a non-ready state for longer than 5 minutes.
Firing 36:
  State:       Resolved
  Started:     2026-10-15T13:14:00Z
  Ended:       2026-10-15T13:40:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-disk-csi-driver-controller-77857b488-k7nvl", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-disk-csi-driver-controller-77857b488-k7nvl has been in a non-ready state for longer than 5 minutes.
Firing 37:
  State:       Resolved
  Started:     2026-10-15T13:14:00Z
  Ended:       2026-10-15T13:40:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-b847b98c9-mnqrr", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-b847b98c9-mnqrr has been in a non-ready state for longer than 5 minutes.
Firing 38:
  State:       Resolved
  Started:     2026-10-15T13:14:00Z
  Ended:       2026-10-15T13:36:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-6d749dbd59-tch26", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-6d749dbd59-tch26 has been in a non-ready state for longer than 5 minutes.
Firing 39:
  State:       Resolved
  Started:     2026-10-15T13:14:00Z
  Ended:       2026-10-15T13:40:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-disk-csi-driver-controller-77857b488-7tv6l", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-disk-csi-driver-controller-77857b488-7tv6l has been in a non-ready state for longer than 5 minutes.
Firing 40:
  State:       Resolved
  Started:     2026-10-15T13:15:00Z
  Ended:       2026-10-15T13:35:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-disk-csi-driver-controller-77857b488-82k6n", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-disk-csi-driver-controller-77857b488-82k6n has been in a non-ready state for longer than 5 minutes.
Firing 41:
  State:       Resolved
  Started:     2026-10-15T13:15:00Z
  Ended:       2026-10-15T13:36:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n", pod="azure-file-csi-driver-controller-b847b98c9-jsqdh", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7vbfo8mc7hb5tobvflpiui7a842m1-q0m2k0e9y1z7s2n/azure-file-csi-driver-controller-b847b98c9-jsqdh has been in a non-ready state for longer than 5 minutes.
Firing 42:
  State:       Resolved
  Started:     2026-10-15T13:15:00Z
  Ended:       2026-10-15T13:37:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5813376-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf7v6diupt5grtmeo5qd4ocn7bfci0v-d8n3d6k7c7h0h4f", pod="csi-snapshot-controller-f45b86c9d-rfmc6", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf7v6diupt5grtmeo5qd4ocn7bfci0v-d8n3d6k7c7h0h4f/csi-snapshot-controller-f45b86c9d-rfmc6 has been in a non-ready state for longer than 5 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 607.834s.
Expected success, but got an error:
    <*fmt.wrapError | 0x1b1ea5606860>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);
    let _endTime = datetime(2026-10-15T13:36:27.6218063Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-nzl7t56rlmwg'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-15T13:36:43.8312817Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07", ActivityId:"20e50d07-291a-4b8a-941b-eeee87c2d5bb", SubActivityId:"20e50d07-291a-4b8a-941b-eeee87c2d5bb", ActivityType:"GW.Http.CallContext", ParentActivityId:"20e50d07-291a-4b8a-941b-eeee87c2d5bb", ActivityStack:"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb > GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T13:36:43.8312817Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\", ActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", SubActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityStack:\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb > GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x1b1ea82ed560>{
            errs: [
                <*fmt.wrapError | 0x1b1ea56067e0>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T13:36:43.8312817Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\", ActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", SubActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityStack:\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb > GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x1b1ea56067c0>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T13:36:43.8312817Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\", ActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", SubActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityStack:\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb > GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x1b1ea4c841e0>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T13:36:43.8312817Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\", ActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", SubActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityStack:\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb > GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x1b1ea4c84000>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T13:36:43.8312817Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\", ActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", SubActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityStack:\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb > GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x1b1ea51ea200>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 607.834s.
Expected success, but got an error:
    <*fmt.wrapError | 0x1b1ea5606860>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);
    let _endTime = datetime(2026-10-15T13:36:27.6218063Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-nzl7t56rlmwg'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-15T13:36:43.8312817Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07", ActivityId:"20e50d07-291a-4b8a-941b-eeee87c2d5bb", SubActivityId:"20e50d07-291a-4b8a-941b-eeee87c2d5bb", ActivityType:"GW.Http.CallContext", ParentActivityId:"20e50d07-291a-4b8a-941b-eeee87c2d5bb", ActivityStack:"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb > GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T13:36:43.8312817Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\", ActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", SubActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityStack:\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb > GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x1b1ea82ed560>{
            errs: [
                <*fmt.wrapError | 0x1b1ea56067e0>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T13:36:43.8312817Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\", ActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", SubActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityStack:\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb > GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x1b1ea56067c0>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T13:36:43.8312817Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\", ActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", SubActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityStack:\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb > GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x1b1ea4c841e0>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T13:36:43.8312817Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\", ActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", SubActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityStack:\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb > GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x1b1ea4c84000>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T13:36:43.8312817Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\", ActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", SubActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityStack:\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb > GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x1b1ea51ea200>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }

alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T14:03:15Z
  Severity:    Sev3
  Labels:      alertname="LeaderElectionLeaseStale", cluster="ci01-j2905984-mgmt-1", component="leader-election", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.170:8080", job="kube-state-metrics", lease="work-manager", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2905984/providers/microsoft.monitor/accounts/services-j2905984", namespace="klusterlet-2tf7t06orjh1refvkt10u0kf3hflts6m", pod="arohcp-monitor-kube-state-metrics-7988dc648c-5gwc4", prometheus="prometheus/prometheus", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Leader election lease work-manager in namespace klusterlet-2tf7t06orjh1refvkt10u0kf3hflts6m on cluster ci01-j2905984-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool "np-s55fk7" for cluster "cluster-candidate-4-21-s55fk7"
Unexpected error:
    <*fmt.wrapError | 0x3ab3a8729d20>: 
    failed to create NodePool np-s55fk7: failed waiting for nodepool="np-s55fk7" for cluster "cluster-candidate-4-21-s55fk7" in resourcegroup="rg-candidate-4-21-2b9bg5zzlrb2" to finish creating: GET https://rp.j2820352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e683b7a-2e40-4525-bc94-88ec837f066b
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e683b7a-2e40-4525-bc94-88ec837f066b",
      "name": "1e683b7a-2e40-4525-bc94-88ec837f066b",
      "status": "Failed",
      "startTime": "2026-10-15T12:48:55.693619679Z",
      "endTime": "2026-10-15T13:18:02.353707633Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine x4g7g8b3n8d8s2y-np-s55fk7-pk7bh-6m8wc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-2b9bg5zzlrb2-candidate-s55fk7--managed/providers/Microsoft.Compute/virtualMachines/x4g7g8b3n8d8s2y-np-s55fk7-pk7bh-6m8wc to exist\nMachine x4g7g8b3n8d8s2y-np-s55fk7-pk7bh-l9fgv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-2b9bg5zzlrb2-candidate-s55fk7--managed/providers/Microsoft.Compute/virtualMachines/x4g7g8b3n8d8s2y-np-s55fk7-pk7bh-l9fgv to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine x4g7g8b3n8d8s2y-np-s55fk7-pk7bh-6m8wc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-2b9bg5zzlrb2-candidate-s55fk7--managed/providers/Microsoft.Compute/virtualMachines/x4g7g8b3n8d8s2y-np-s55fk7-pk7bh-6m8wc to exist\nMachine x4g7g8b3n8d8s2y-np-s55fk7-pk7bh-l9fgv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-2b9bg5zzlrb2-candidate-s55fk7--managed/providers/Microsoft.Compute/virtualMachines/x4g7g8b3n8d8s2y-np-s55fk7-pk7bh-l9fgv to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2820352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e683b7a-2e40-4525-bc94-88ec837f066b
            correlation ID: 9443cd05-b916-4412-9159-06a0bacd7ec5
            client request ID: 4253deed-8a8e-4609-a015-aff150f302ca
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool "np-s55fk7" for cluster "cluster-candidate-4-21-s55fk7"
Unexpected error:
    <*fmt.wrapError | 0x3ab3a8729d20>: 
    failed to create NodePool np-s55fk7: failed waiting for nodepool="np-s55fk7" for cluster "cluster-candidate-4-21-s55fk7" in resourcegroup="rg-candidate-4-21-2b9bg5zzlrb2" to finish creating: GET https://rp.j2820352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e683b7a-2e40-4525-bc94-88ec837f066b
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e683b7a-2e40-4525-bc94-88ec837f066b",
      "name": "1e683b7a-2e40-4525-bc94-88ec837f066b",
      "status": "Failed",
      "startTime": "2026-10-15T12:48:55.693619679Z",
      "endTime": "2026-10-15T13:18:02.353707633Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine x4g7g8b3n8d8s2y-np-s55fk7-pk7bh-6m8wc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-2b9bg5zzlrb2-candidate-s55fk7--managed/providers/Microsoft.Compute/virtualMachines/x4g7g8b3n8d8s2y-np-s55fk7-pk7bh-6m8wc to exist\nMachine x4g7g8b3n8d8s2y-np-s55fk7-pk7bh-l9fgv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-2b9bg5zzlrb2-candidate-s55fk7--managed/providers/Microsoft.Compute/virtualMachines/x4g7g8b3n8d8s2y-np-s55fk7-pk7bh-l9fgv to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine x4g7g8b3n8d8s2y-np-s55fk7-pk7bh-6m8wc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-2b9bg5zzlrb2-candidate-s55fk7--managed/providers/Microsoft.Compute/virtualMachines/x4g7g8b3n8d8s2y-np-s55fk7-pk7bh-6m8wc to exist\nMachine x4g7g8b3n8d8s2y-np-s55fk7-pk7bh-l9fgv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-2b9bg5zzlrb2-candidate-s55fk7--managed/providers/Microsoft.Compute/virtualMachines/x4g7g8b3n8d8s2y-np-s55fk7-pk7bh-l9fgv to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2820352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e683b7a-2e40-4525-bc94-88ec837f066b
            correlation ID: 9443cd05-b916-4412-9159-06a0bacd7ec5
            client request ID: 4253deed-8a8e-4609-a015-aff150f302ca
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-94d2g5wkbdwn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x3b3acad40780>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-94d2g5wkbdwn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9276160-mgmt-1/ocm-arohcpci01-2tf7qg3qc9tr83rniscs197k8bo2fac2-o2o1a7a4r7x5y6r: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-94d2g5wkbdwn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9276160-mgmt-1/ocm-arohcpci01-2tf7qg3qc9tr83rniscs197k8bo2fac2-o2o1a7a4r7x5y6r: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*errors.errorString | 0x3b3acbd1bd90>{
            s: "dev/westus3/ci01-j9276160-mgmt-1/ocm-arohcpci01-2tf7qg3qc9tr83rniscs197k8bo2fac2-o2o1a7a4r7x5y6r: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-94d2g5wkbdwn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x3b3acad40780>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-94d2g5wkbdwn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9276160-mgmt-1/ocm-arohcpci01-2tf7qg3qc9tr83rniscs197k8bo2fac2-o2o1a7a4r7x5y6r: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-94d2g5wkbdwn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9276160-mgmt-1/ocm-arohcpci01-2tf7qg3qc9tr83rniscs197k8bo2fac2-o2o1a7a4r7x5y6r: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*errors.errorString | 0x3b3acbd1bd90>{
            s: "dev/westus3/ci01-j9276160-mgmt-1/ocm-arohcpci01-2tf7qg3qc9tr83rniscs197k8bo2fac2-o2o1a7a4r7x5y6r: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        },
    }
occurred

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T13:08:03Z
  Ended:       2026-10-15T13:18:04Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j4144640-svc", component="backend", name="csstatedump", region="westus3", severity="warning"
  Description: Backend controller workqueue csstatedump has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

time=2026-10-15T11:32:57.186Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description="Step cluster\n  Kind: ARM\n  Template: templates/svc-cluster.bicep\n  Parameters: configurations/svc-cluster.tmpl.bicepparam"
time=2026-10-15T11:32:58.546Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster
time=2026-10-15T11:32:59.802Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=efd05607e6a807430e268f57c7b15b6ec1dffd0e4888fc1e3250ffadf0c2b8e4 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7048832-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fefd05607e6a807430e268f57c7b15b6ec1dffd0e4888fc1e3250ffadf0c2b8e4
time=2026-10-15T11:35:30.201Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7048832-svc/providers/Microsoft.Resources/deployments/efd05607e6a807430e268f57c7b15b6ec1dffd0e4888fc1e3250ffadf0c2b8e4/operationStatuses/08584095429065032552\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7048832-svc/providers/Microsoft.Resources/deployments/cluster-p5nidgkpcp2bc\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7048832-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j7048832-svc\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\",\\r\\n                \\\"message\\\": \\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\"https://ah-ci01-se-j7048832-1.vault.azure.net\\\\\\\"\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x31dfe7cead80>: 
    failed to create NodePool np-delete-main: failed waiting for nodepool="np-delete-main" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-w4sdcc2f9b98" to finish creating: GET https://rp.j9466368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1a9a190b-4bfc-40cb-9dd3-61f444b3d45a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1a9a190b-4bfc-40cb-9dd3-61f444b3d45a",
      "name": "1a9a190b-4bfc-40cb-9dd3-61f444b3d45a",
      "status": "Failed",
      "startTime": "2026-10-15T11:58:54.751404091Z",
      "endTime": "2026-10-15T12:28:03.314684153Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine v0x6h3c4b5g9z6x-np-delete-main-cv9ln-97hs2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-w4sdcc2f9b98--managed/providers/Microsoft.Compute/virtualMachines/v0x6h3c4b5g9z6x-np-delete-main-cv9ln-97hs2 to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine v0x6h3c4b5g9z6x-np-delete-main-cv9ln-97hs2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-w4sdcc2f9b98--managed/providers/Microsoft.Compute/virtualMachines/v0x6h3c4b5g9z6x-np-delete-main-cv9ln-97hs2 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9466368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1a9a190b-4bfc-40cb-9dd3-61f444b3d45a
            correlation ID: a5182798-ac67-4625-a997-1317fca7e220
            client request ID: d5f9fdbc-cb7e-4972-bee6-661a76843384
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x31dfe7cead80>: 
    failed to create NodePool np-delete-main: failed waiting for nodepool="np-delete-main" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-w4sdcc2f9b98" to finish creating: GET https://rp.j9466368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1a9a190b-4bfc-40cb-9dd3-61f444b3d45a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1a9a190b-4bfc-40cb-9dd3-61f444b3d45a",
      "name": "1a9a190b-4bfc-40cb-9dd3-61f444b3d45a",
      "status": "Failed",
      "startTime": "2026-10-15T11:58:54.751404091Z",
      "endTime": "2026-10-15T12:28:03.314684153Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine v0x6h3c4b5g9z6x-np-delete-main-cv9ln-97hs2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-w4sdcc2f9b98--managed/providers/Microsoft.Compute/virtualMachines/v0x6h3c4b5g9z6x-np-delete-main-cv9ln-97hs2 to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine v0x6h3c4b5g9z6x-np-delete-main-cv9ln-97hs2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-w4sdcc2f9b98--managed/providers/Microsoft.Compute/virtualMachines/v0x6h3c4b5g9z6x-np-delete-main-cv9ln-97hs2 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9466368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1a9a190b-4bfc-40cb-9dd3-61f444b3d45a
            correlation ID: a5182798-ac67-4625-a997-1317fca7e220
            client request ID: d5f9fdbc-cb7e-4972-bee6-661a76843384

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES
Unexpected error:
    <*fmt.wrapError | 0x30c7dd043d80>: 
    failed to create NodePool des-np: failed waiting for nodepool="des-np" for cluster "des-encrypt" in resourcegroup="des-encrypt-bkhxjv72lv2h" to finish creating: GET https://rp.j5369728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a2658dd-7908-49f0-96ce-e3959f8275fe
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a2658dd-7908-49f0-96ce-e3959f8275fe",
      "name": "4a2658dd-7908-49f0-96ce-e3959f8275fe",
      "status": "Failed",
      "startTime": "2026-10-15T11:58:55.879104397Z",
      "endTime": "2026-10-15T12:27:56.020229321Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine des-encrypt-des-np-nv5pg-24w7d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-bkhxjv72lv2h--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-nv5pg-24w7d to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine des-encrypt-des-np-nv5pg-24w7d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-bkhxjv72lv2h--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-nv5pg-24w7d to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5369728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a2658dd-7908-49f0-96ce-e3959f8275fe
            correlation ID: 89998081-e643-4b0b-aa81-6a2420c250ac
            client request ID: 66cbea70-ac76-440c-ab1a-21754876dae9
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES
Unexpected error:
    <*fmt.wrapError | 0x30c7dd043d80>: 
    failed to create NodePool des-np: failed waiting for nodepool="des-np" for cluster "des-encrypt" in resourcegroup="des-encrypt-bkhxjv72lv2h" to finish creating: GET https://rp.j5369728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a2658dd-7908-49f0-96ce-e3959f8275fe
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a2658dd-7908-49f0-96ce-e3959f8275fe",
      "name": "4a2658dd-7908-49f0-96ce-e3959f8275fe",
      "status": "Failed",
      "startTime": "2026-10-15T11:58:55.879104397Z",
      "endTime": "2026-10-15T12:27:56.020229321Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine des-encrypt-des-np-nv5pg-24w7d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-bkhxjv72lv2h--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-nv5pg-24w7d to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine des-encrypt-des-np-nv5pg-24w7d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-bkhxjv72lv2h--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-nv5pg-24w7d to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5369728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a2658dd-7908-49f0-96ce-e3959f8275fe
            correlation ID: 89998081-e643-4b0b-aa81-6a2420c250ac
            client request ID: 66cbea70-ac76-440c-ab1a-21754876dae9
occurred

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T11:47:03Z
  Ended:       2026-10-15T12:01:00Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j7192576-svc", component="backend", name="clusterresources", region="centralus", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:
    <*errors.joinError | 0x1ff649812120>: 
    resource group "rg-candidate-4-20-ffzv544l6w92": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-4-20-pf2tq2" in resourcegroup="rg-candidate-4-20-ffzv544l6w92" to finish deleting: GET https://rp.j4344192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/08aca983-942f-491c-b751-ff15b4ba5c58
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/08aca983-942f-491c-b751-ff15b4ba5c58",
      "name": "08aca983-942f-491c-b751-ff15b4ba5c58",
      "status": "Failed",
      "startTime": "2026-10-15T11:21:34.419311778Z",
      "endTime": "2026-10-15T11:45:37.398768071Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4344192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/08aca983-942f-491c-b751-ff15b4ba5c58
            correlation ID: 34b324bb-a098-46c8-8e16-84d85878762f
            client request ID: 039d8e60-0e2e-4da7-ae0f-2e1857606679
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:
    <*errors.joinError | 0x1ff649812120>: 
    resource group "rg-candidate-4-20-ffzv544l6w92": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-4-20-pf2tq2" in resourcegroup="rg-candidate-4-20-ffzv544l6w92" to finish deleting: GET https://rp.j4344192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/08aca983-942f-491c-b751-ff15b4ba5c58
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/08aca983-942f-491c-b751-ff15b4ba5c58",
      "name": "08aca983-942f-491c-b751-ff15b4ba5c58",
      "status": "Failed",
      "startTime": "2026-10-15T11:21:34.419311778Z",
      "endTime": "2026-10-15T11:45:37.398768071Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4344192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/08aca983-942f-491c-b751-ff15b4ba5c58
            correlation ID: 34b324bb-a098-46c8-8e16-84d85878762f
            client request ID: 039d8e60-0e2e-4da7-ae0f-2e1857606679
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool "np-1" for private ingress cluster "private-ingress"
Unexpected error:
    <*fmt.wrapError | 0x315b320b0100>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "private-ingress" in resourcegroup="private-ingress-mjm476sds2q9" to finish creating: GET https://rp.j4344192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/be19e499-8b2d-4dc2-9a9d-968d043575f5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/be19e499-8b2d-4dc2-9a9d-968d043575f5",
      "name": "be19e499-8b2d-4dc2-9a9d-968d043575f5",
      "status": "Failed",
      "startTime": "2026-10-15T11:09:27.202594199Z",
      "endTime": "2026-10-15T11:38:27.187446227Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine private-ingress-np-1-pkg2x-wsc5s: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-mjm476sds2q9--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-pkg2x-wsc5s to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine private-ingress-np-1-pkg2x-wsc5s: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-mjm476sds2q9--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-pkg2x-wsc5s to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4344192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/be19e499-8b2d-4dc2-9a9d-968d043575f5
            correlation ID: 36e13d07-8647-4ed3-96df-c8b4fe4cea15
            client request ID: 9d12f59b-3610-4a88-aed2-e3b9cf9f6a2e
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool "np-1" for private ingress cluster "private-ingress"
Unexpected error:
    <*fmt.wrapError | 0x315b320b0100>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "private-ingress" in resourcegroup="private-ingress-mjm476sds2q9" to finish creating: GET https://rp.j4344192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/be19e499-8b2d-4dc2-9a9d-968d043575f5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/be19e499-8b2d-4dc2-9a9d-968d043575f5",
      "name": "be19e499-8b2d-4dc2-9a9d-968d043575f5",
      "status": "Failed",
      "startTime": "2026-10-15T11:09:27.202594199Z",
      "endTime": "2026-10-15T11:38:27.187446227Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine private-ingress-np-1-pkg2x-wsc5s: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-mjm476sds2q9--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-pkg2x-wsc5s to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine private-ingress-np-1-pkg2x-wsc5s: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-mjm476sds2q9--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-pkg2x-wsc5s to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4344192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/be19e499-8b2d-4dc2-9a9d-968d043575f5
            correlation ID: 36e13d07-8647-4ed3-96df-c8b4fe4cea15
            client request ID: 9d12f59b-3610-4a88-aed2-e3b9cf9f6a2e
occurred

alert fired 15 time(s)
Firing 1:
  State:       Resolved
  Duration:    18m2.3532689s
  Started:     2026-10-15T11:28:00Z
  Ended:       2026-10-15T11:46:03Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4344192-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u", pod="ignition-server-proxy-96899d89b-dsgxl", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-dsgxl has been in a non-ready state for longer than 5 minutes.
  Expression:  sum by (namespace, pod, cluster, region) (max by (namespace, pod, cluster, region) (kube_pod_status_phase{job="kube-state-metrics",namespace!~"klusterlet-.*",phase=~"Pending|Unknown|Failed",prometheus="prometheus/prometheus"}) * on (namespace, pod, cluster) group_left (owner_kind) topk by (namespace, pod, cluster, region) (1, max by (namespace, pod, owner_kind, cluster, region) (kube_pod_owner{owner_kind!="Job",prometheus="prometheus/prometheus"}))) > 0 unless on (namespace, pod, cluster) kube_pod_deletion_timestamp
  Annotation description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-dsgxl has been in a non-ready state for longer than 5 minutes.
  Summary:     Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-dsgxl has been in a non-ready state for more than 5 minutes.
  Runbook URL: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready
Firing 2:
  State:       Resolved
  Duration:    21m58.9502658s
  Started:     2026-10-15T11:29:02Z
  Ended:       2026-10-15T11:51:01Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4344192-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p", pod="multus-admission-controller-7768fc485b-p9zx2", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p/multus-admission-controller-7768fc485b-p9zx2 has been in a non-ready state for longer than 5 minutes.
  Expression:  sum by (namespace, pod, cluster, region) (max by (namespace, pod, cluster, region) (kube_pod_status_phase{job="kube-state-metrics",namespace!~"klusterlet-.*",phase=~"Pending|Unknown|Failed",prometheus="prometheus/prometheus"}) * on (namespace, pod, cluster) group_left (owner_kind) topk by (namespace, pod, cluster, region) (1, max by (namespace, pod, owner_kind, cluster, region) (kube_pod_owner{owner_kind!="Job",prometheus="prometheus/prometheus"}))) > 0 unless on (namespace, pod, cluster) kube_pod_deletion_timestamp
  Annotation description: Pod ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p/multus-admission-controller-7768fc485b-p9zx2 has been in a non-ready state for longer than 5 minutes.
  Summary:     Pod ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p/multus-admission-controller-7768fc485b-p9zx2 has been in a non-ready state for more than 5 minutes.
  Runbook URL: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready
Firing 3:
  State:       Resolved
  Duration:    21m58.7017765s
  Started:     2026-10-15T11:29:02Z
  Ended:       2026-10-15T11:51:00Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4344192-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p", pod="multus-admission-controller-7768fc485b-qs2w5", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p/multus-admission-controller-7768fc485b-qs2w5 has been in a non-ready state for longer than 5 minutes.
  Expression:  sum by (namespace, pod, cluster, region) (max by (namespace, pod, cluster, region) (kube_pod_status_phase{job="kube-state-metrics",namespace!~"klusterlet-.*",phase=~"Pending|Unknown|Failed",prometheus="prometheus/prometheus"}) * on (namespace, pod, cluster) group_left (owner_kind) topk by (namespace, pod, cluster, region) (1, max by (namespace, pod, owner_kind, cluster, region) (kube_pod_owner{owner_kind!="Job",prometheus="prometheus/prometheus"}))) > 0 unless on (namespace, pod, cluster) kube_pod_deletion_timestamp
  Annotation description: Pod ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p/multus-admission-controller-7768fc485b-qs2w5 has been in a non-ready state for longer than 5 minutes.
  Summary:     Pod ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p/multus-admission-controller-7768fc485b-qs2w5 has been in a non-ready state for more than 5 minutes.
  Runbook URL: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready
Firing 4:
  State:       Resolved
  Duration:    13m0.1696606s
  Started:     2026-10-15T11:29:02Z
  Ended:       2026-10-15T11:42:02Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4344192-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u", pod="ignition-server-proxy-96899d89b-7s88k", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-7s88k has been in a non-ready state for longer than 5 minutes.
  Expression:  sum by (namespace, pod, cluster, region) (max by (namespace, pod, cluster, region) (kube_pod_status_phase{job="kube-state-metrics",namespace!~"klusterlet-.*",phase=~"Pending|Unknown|Failed",prometheus="prometheus/prometheus"}) * on (namespace, pod, cluster) group_left (owner_kind) topk by (namespace, pod, cluster, region) (1, max by (namespace, pod, owner_kind, cluster, region) (kube_pod_owner{owner_kind!="Job",prometheus="prometheus/prometheus"}))) > 0 unless on (namespace, pod, cluster) kube_pod_deletion_timestamp
  Annotation description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-7s88k has been in a non-ready state for longer than 5 minutes.
  Summary:     Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-7s88k has been in a non-ready state for more than 5 minutes.
  Runbook URL: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready
Firing 5:
  State:       Resolved
  Duration:    13m0.1165754s
  Started:     2026-10-15T11:29:02Z
  Ended:       2026-10-15T11:42:02Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4344192-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u", pod="ignition-server-proxy-96899d89b-m7c7t", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-m7c7t has been in a non-ready state for longer than 5 minutes.
  Expression:  sum by (namespace, pod, cluster, region) (max by (namespace, pod, cluster, region) (kube_pod_status_phase{job="kube-state-metrics",namespace!~"klusterlet-.*",phase=~"Pending|Unknown|Failed",prometheus="prometheus/prometheus"}) * on (namespace, pod, cluster) group_left (owner_kind) topk by (namespace, pod, cluster, region) (1, max by (namespace, pod, owner_kind, cluster, region) (kube_pod_owner{owner_kind!="Job",prometheus="prometheus/prometheus"}))) > 0 unless on (namespace, pod, cluster) kube_pod_deletion_timestamp
  Annotation description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-m7c7t has been in a non-ready state for longer than 5 minutes.
  Summary:     Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-m7c7t has been in a non-ready state for more than 5 minutes.
  Runbook URL: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready
Firing 6:
  State:       Resolved
  Duration:    21m58.6099069s
  Started:     2026-10-15T11:29:02Z
  Ended:       2026-10-15T11:51:01Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4344192-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p", pod="multus-admission-controller-7768fc485b-8skmn", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p/multus-admission-controller-7768fc485b-8skmn has been in a non-ready state for longer than 5 minutes.
  Expression:  sum by (namespace, pod, cluster, region) (max by (namespace, pod, cluster, region) (kube_pod_status_phase{job="kube-state-metrics",namespace!~"klusterlet-.*",phase=~"Pending|Unknown|Failed",prometheus="prometheus/prometheus"}) * on (namespace, pod, cluster) group_left (owner_kind) topk by (namespace, pod, cluster, region) (1, max by (namespace, pod, owner_kind, cluster, region) (kube_pod_owner{owner_kind!="Job",prometheus="prometheus/prometheus"}))) > 0 unless on (namespace, pod, cluster) kube_pod_deletion_timestamp
  Annotation description: Pod ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p/multus-admission-controller-7768fc485b-8skmn has been in a non-ready state for longer than 5 minutes.
  Summary:     Pod ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p/multus-admission-controller-7768fc485b-8skmn has been in a non-ready state for more than 5 minutes.
  Runbook URL: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready
Firing 7:
  State:       Resolved
  Duration:    12m1.6834771s
  Started:     2026-10-15T11:30:00Z
  Ended:       2026-10-15T11:42:02Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4344192-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u", pod="ignition-server-proxy-96899d89b-qd2fm", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-qd2fm has been in a non-ready state for longer than 5 minutes.
  Expression:  sum by (namespace, pod, cluster, region) (max by (namespace, pod, cluster, region) (kube_pod_status_phase{job="kube-state-metrics",namespace!~"klusterlet-.*",phase=~"Pending|Unknown|Failed",prometheus="prometheus/prometheus"}) * on (namespace, pod, cluster) group_left (owner_kind) topk by (namespace, pod, cluster, region) (1, max by (namespace, pod, owner_kind, cluster, region) (kube_pod_owner{owner_kind!="Job",prometheus="prometheus/prometheus"}))) > 0 unless on (namespace, pod, cluster) kube_pod_deletion_timestamp
  Annotation description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-qd2fm has been in a non-ready state for longer than 5 minutes.
  Summary:     Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-qd2fm has been in a non-ready state for more than 5 minutes.
  Runbook URL: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready
Firing 8:
  State:       Resolved
  Duration:    21m0.2690359s
  Started:     2026-10-15T11:30:00Z
  Ended:       2026-10-15T11:51:01Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4344192-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p", pod="multus-admission-controller-7768fc485b-f7z4w", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p/multus-admission-controller-7768fc485b-f7z4w has been in a non-ready state for longer than 5 minutes.
  Expression:  sum by (namespace, pod, cluster, region) (max by (namespace, pod, cluster, region) (kube_pod_status_phase{job="kube-state-metrics",namespace!~"klusterlet-.*",phase=~"Pending|Unknown|Failed",prometheus="prometheus/prometheus"}) * on (namespace, pod, cluster) group_left (owner_kind) topk by (namespace, pod, cluster, region) (1, max by (namespace, pod, owner_kind, cluster, region) (kube_pod_owner{owner_kind!="Job",prometheus="prometheus/prometheus"}))) > 0 unless on (namespace, pod, cluster) kube_pod_deletion_timestamp
  Annotation description: Pod ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p/multus-admission-controller-7768fc485b-f7z4w has been in a non-ready state for longer than 5 minutes.
  Summary:     Pod ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p/multus-admission-controller-7768fc485b-f7z4w has been in a non-ready state for more than 5 minutes.
  Runbook URL: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready
Firing 9:
  State:       Resolved
  Duration:    12m1.7335379s
  Started:     2026-10-15T11:30:01Z
  Ended:       2026-10-15T11:42:02Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4344192-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u", pod="ignition-server-proxy-96899d89b-69ppq", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-69ppq has been in a non-ready state for longer than 5 minutes.
  Expression:  sum by (namespace, pod, cluster, region) (max by (namespace, pod, cluster, region) (kube_pod_status_phase{job="kube-state-metrics",namespace!~"klusterlet-.*",phase=~"Pending|Unknown|Failed",prometheus="prometheus/prometheus"}) * on (namespace, pod, cluster) group_left (owner_kind) topk by (namespace, pod, cluster, region) (1, max by (namespace, pod, owner_kind, cluster, region) (kube_pod_owner{owner_kind!="Job",prometheus="prometheus/prometheus"}))) > 0 unless on (namespace, pod, cluster) kube_pod_deletion_timestamp
  Annotation description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-69ppq has been in a non-ready state for longer than 5 minutes.
  Summary:     Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-69ppq has been in a non-ready state for more than 5 minutes.
  Runbook URL: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready
Firing 10:
  State:       Resolved
  Duration:    16m1.9642414s
  Started:     2026-10-15T11:30:01Z
  Ended:       2026-10-15T11:46:03Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4344192-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u", pod="ignition-server-proxy-96899d89b-g7krp", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-g7krp has been in a non-ready state for longer than 5 minutes.
  Expression:  sum by (namespace, pod, cluster, region) (max by (namespace, pod, cluster, region) (kube_pod_status_phase{job="kube-state-metrics",namespace!~"klusterlet-.*",phase=~"Pending|Unknown|Failed",prometheus="prometheus/prometheus"}) * on (namespace, pod, cluster) group_left (owner_kind) topk by (namespace, pod, cluster, region) (1, max by (namespace, pod, owner_kind, cluster, region) (kube_pod_owner{owner_kind!="Job",prometheus="prometheus/prometheus"}))) > 0 unless on (namespace, pod, cluster) kube_pod_deletion_timestamp
  Annotation description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-g7krp has been in a non-ready state for longer than 5 minutes.
  Summary:     Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-g7krp has been in a non-ready state for more than 5 minutes.
  Runbook URL: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready
Firing 11:
  State:       Resolved
  Duration:    16m1.8339648s
  Started:     2026-10-15T11:30:01Z
  Ended:       2026-10-15T11:46:03Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4344192-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u", pod="ignition-server-proxy-96899d89b-kzvzm", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-kzvzm has been in a non-ready state for longer than 5 minutes.
  Expression:  sum by (namespace, pod, cluster, region) (max by (namespace, pod, cluster, region) (kube_pod_status_phase{job="kube-state-metrics",namespace!~"klusterlet-.*",phase=~"Pending|Unknown|Failed",prometheus="prometheus/prometheus"}) * on (namespace, pod, cluster) group_left (owner_kind) topk by (namespace, pod, cluster, region) (1, max by (namespace, pod, owner_kind, cluster, region) (kube_pod_owner{owner_kind!="Job",prometheus="prometheus/prometheus"}))) > 0 unless on (namespace, pod, cluster) kube_pod_deletion_timestamp
  Annotation description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-kzvzm has been in a non-ready state for longer than 5 minutes.
  Summary:     Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-kzvzm has been in a non-ready state for more than 5 minutes.
  Runbook URL: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready
Firing 12:
  State:       Resolved
  Duration:    9m59.2825786s
  Started:     2026-10-15T11:30:01Z
  Ended:       2026-10-15T11:40:00Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4344192-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u", pod="ignition-server-proxy-96899d89b-jj58z", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-jj58z has been in a non-ready state for longer than 5 minutes.
  Expression:  sum by (namespace, pod, cluster, region) (max by (namespace, pod, cluster, region) (kube_pod_status_phase{job="kube-state-metrics",namespace!~"klusterlet-.*",phase=~"Pending|Unknown|Failed",prometheus="prometheus/prometheus"}) * on (namespace, pod, cluster) group_left (owner_kind) topk by (namespace, pod, cluster, region) (1, max by (namespace, pod, owner_kind, cluster, region) (kube_pod_owner{owner_kind!="Job",prometheus="prometheus/prometheus"}))) > 0 unless on (namespace, pod, cluster) kube_pod_deletion_timestamp
  Annotation description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-jj58z has been in a non-ready state for longer than 5 minutes.
  Summary:     Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-jj58z has been in a non-ready state for more than 5 minutes.
  Runbook URL: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready
Firing 13:
  State:       Resolved
  Duration:    16m1.4449019s
  Started:     2026-10-15T11:30:01Z
  Ended:       2026-10-15T11:46:03Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4344192-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u", pod="ignition-server-proxy-96899d89b-frrtn", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-frrtn has been in a non-ready state for longer than 5 minutes.
  Expression:  sum by (namespace, pod, cluster, region) (max by (namespace, pod, cluster, region) (kube_pod_status_phase{job="kube-state-metrics",namespace!~"klusterlet-.*",phase=~"Pending|Unknown|Failed",prometheus="prometheus/prometheus"}) * on (namespace, pod, cluster) group_left (owner_kind) topk by (namespace, pod, cluster, region) (1, max by (namespace, pod, owner_kind, cluster, region) (kube_pod_owner{owner_kind!="Job",prometheus="prometheus/prometheus"}))) > 0 unless on (namespace, pod, cluster) kube_pod_deletion_timestamp
  Annotation description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-frrtn has been in a non-ready state for longer than 5 minutes.
  Summary:     Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-frrtn has been in a non-ready state for more than 5 minutes.
  Runbook URL: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready
Firing 14:
  State:       Resolved
  Duration:    16m1.2906049s
  Started:     2026-10-15T11:30:01Z
  Ended:       2026-10-15T11:46:03Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4344192-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u", pod="ignition-server-proxy-96899d89b-85wjz", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-85wjz has been in a non-ready state for longer than 5 minutes.
  Expression:  sum by (namespace, pod, cluster, region) (max by (namespace, pod, cluster, region) (kube_pod_status_phase{job="kube-state-metrics",namespace!~"klusterlet-.*",phase=~"Pending|Unknown|Failed",prometheus="prometheus/prometheus"}) * on (namespace, pod, cluster) group_left (owner_kind) topk by (namespace, pod, cluster, region) (1, max by (namespace, pod, owner_kind, cluster, region) (kube_pod_owner{owner_kind!="Job",prometheus="prometheus/prometheus"}))) > 0 unless on (namespace, pod, cluster) kube_pod_deletion_timestamp
  Annotation description: Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-85wjz has been in a non-ready state for longer than 5 minutes.
  Summary:     Pod ocm-arohcpci01-2tf695auvop30sp3uvqbi25u876gkohp-k7a5h2j7n8o1n9u/ignition-server-proxy-96899d89b-85wjz has been in a non-ready state for more than 5 minutes.
  Runbook URL: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready
Firing 15:
  State:       Resolved
  Duration:    20m59.0228848s
  Started:     2026-10-15T11:30:01Z
  Ended:       2026-10-15T11:51:00Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4344192-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p", pod="multus-admission-controller-7768fc485b-lgtjg", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p/multus-admission-controller-7768fc485b-lgtjg has been in a non-ready state for longer than 5 minutes.
  Expression:  sum by (namespace, pod, cluster, region) (max by (namespace, pod, cluster, region) (kube_pod_status_phase{job="kube-state-metrics",namespace!~"klusterlet-.*",phase=~"Pending|Unknown|Failed",prometheus="prometheus/prometheus"}) * on (namespace, pod, cluster) group_left (owner_kind) topk by (namespace, pod, cluster, region) (1, max by (namespace, pod, owner_kind, cluster, region) (kube_pod_owner{owner_kind!="Job",prometheus="prometheus/prometheus"}))) > 0 unless on (namespace, pod, cluster) kube_pod_deletion_timestamp
  Annotation description: Pod ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p/multus-admission-controller-7768fc485b-lgtjg has been in a non-ready state for longer than 5 minutes.
  Summary:     Pod ocm-arohcpci01-2tf69cj994sb12ojr5n9i468ie1f849r-i7o9o6x2n4m7m4p/multus-admission-controller-7768fc485b-lgtjg has been in a non-ready state for more than 5 minutes.
  Runbook URL: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready

alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved) [STILL FIRING]
  Duration:    still firing
  Started:     2026-10-15T11:58:01Z
  Ended:       still firing
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j4344192-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.65.3:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4344192/providers/microsoft.monitor/accounts/hcps-j4344192", namespace="ocm-arohcpci01-2tf69jfreidst7lne922ffqmev0pap72-w6l7v6s5o6w5x1n", pod="arohcp-monitor-kube-state-metrics-7988dc648c-g8j7n", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tf69jfreidst7lne922ffqmev0pap72-w6l7v6s5o6w5x1n/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
  Expression:  (kube_deployment_spec_replicas{job="kube-state-metrics"} > kube_deployment_status_replicas_available{job="kube-state-metrics"}) and (changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[10m]) == 0)
  Annotation description: Deployment ocm-arohcpci01-2tf69jfreidst7lne922ffqmev0pap72-w6l7v6s5o6w5x1n/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
  Summary:     Deployment ocm-arohcpci01-2tf69jfreidst7lne922ffqmev0pap72-w6l7v6s5o6w5x1n/cloud-network-config-controller has not matched the expected number of replicas.
  Runbook URL: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentreplicasmismatch
Firing 2:
  State:       Fired (not resolved) [STILL FIRING]
  Duration:    still firing
  Started:     2026-10-15T11:58:01Z
  Ended:       still firing
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j4344192-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.65.3:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4344192/providers/microsoft.monitor/accounts/hcps-j4344192", namespace="ocm-arohcpci01-2tf69jfreidst7lne922ffqmev0pap72-w6l7v6s5o6w5x1n", pod="arohcp-monitor-kube-state-metrics-7988dc648c-g8j7n", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tf69jfreidst7lne922ffqmev0pap72-w6l7v6s5o6w5x1n/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
  Expression:  (kube_deployment_spec_replicas{job="kube-state-metrics"} > kube_deployment_status_replicas_available{job="kube-state-metrics"}) and (changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[10m]) == 0)
  Annotation description: Deployment ocm-arohcpci01-2tf69jfreidst7lne922ffqmev0pap72-w6l7v6s5o6w5x1n/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
  Summary:     Deployment ocm-arohcpci01-2tf69jfreidst7lne922ffqmev0pap72-w6l7v6s5o6w5x1n/cloud-network-config-controller has not matched the expected number of replicas.
  Runbook URL: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentreplicasmismatch

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T11:22:20Z
  Ended:       2026-10-15T11:41:21Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j6365312-svc", component="backend", name="actualhostedcluster", region="centralus", severity="warning"
  Description: Backend controller workqueue actualhostedcluster has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

time=2026-10-15T10:41:48.821Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description="Step cluster-create\n  Kind: Shell\n  Command: ./aks-cluster-create\n"
time=2026-10-15T11:11:48.321Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err="stamp 1: error running Shell Step, failed to execute shell command: {\"time\":\"2026-10-15T10:41:48.83104699Z\",\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\",\"resourceGroup\":\"hcp-underlay-ci01-j0731648-mgmt-1\",\"clusterName\":\"ci01-j0731648-mgmt-1\",\"region\":\"westus3\"}\n{\"time\":\"2026-10-15T10:41:50.721098398Z\",\"level\":\"INFO\",\"msg\":\"configured pools\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\",\"resourceGroup\":\"hcp-underlay-ci01-j0731648-mgmt-1\",\"clusterName\":\"ci01-j0731648-mgmt-1\",\"region\":\"westus3\",\"system\":1,\"worker\":3,\"infra\":2}\n{\"time\":\"2026-10-15T10:41:51.099462414Z\",\"level\":\"INFO\",\"msg\":\"deploying configured resources\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\",\"resourceGroup\":\"hcp-underlay-ci01-j0731648-mgmt-1\",\"clusterName\":\"ci01-j0731648-mgmt-1\",\"region\":\"westus3\",\"deployment\":\"aks-cluster-create\",\"poolCount\":6,\"clusterFields\":null}\n signal: killed"

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x1bbdc5567a00>: 
    failed to create NodePool np-delete-main: failed waiting for nodepool="np-delete-main" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-fdhds6r4wpkt" to finish creating: GET https://rp.j9974912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b1e08d2d-8e5a-4d42-9ec3-fa6f0e7ff42e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b1e08d2d-8e5a-4d42-9ec3-fa6f0e7ff42e",
      "name": "b1e08d2d-8e5a-4d42-9ec3-fa6f0e7ff42e",
      "status": "Failed",
      "startTime": "2026-10-15T10:52:57.652683505Z",
      "endTime": "2026-10-15T11:21:59.5385968Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine f2p5c0p1f1c4g7r-np-delete-main-gpxxw-w6pqj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-fdhds6r4wpkt--managed/providers/Microsoft.Compute/virtualMachines/f2p5c0p1f1c4g7r-np-delete-main-gpxxw-w6pqj to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine f2p5c0p1f1c4g7r-np-delete-main-gpxxw-w6pqj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-fdhds6r4wpkt--managed/providers/Microsoft.Compute/virtualMachines/f2p5c0p1f1c4g7r-np-delete-main-gpxxw-w6pqj to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9974912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b1e08d2d-8e5a-4d42-9ec3-fa6f0e7ff42e
            correlation ID: 2fff4344-7e6c-43c1-b953-d8924fffcf5d
            client request ID: 27cbaf58-6dac-4a2b-a04f-a25864687397
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x1bbdc5567a00>: 
    failed to create NodePool np-delete-main: failed waiting for nodepool="np-delete-main" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-fdhds6r4wpkt" to finish creating: GET https://rp.j9974912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b1e08d2d-8e5a-4d42-9ec3-fa6f0e7ff42e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b1e08d2d-8e5a-4d42-9ec3-fa6f0e7ff42e",
      "name": "b1e08d2d-8e5a-4d42-9ec3-fa6f0e7ff42e",
      "status": "Failed",
      "startTime": "2026-10-15T10:52:57.652683505Z",
      "endTime": "2026-10-15T11:21:59.5385968Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine f2p5c0p1f1c4g7r-np-delete-main-gpxxw-w6pqj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-fdhds6r4wpkt--managed/providers/Microsoft.Compute/virtualMachines/f2p5c0p1f1c4g7r-np-delete-main-gpxxw-w6pqj to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine f2p5c0p1f1c4g7r-np-delete-main-gpxxw-w6pqj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-fdhds6r4wpkt--managed/providers/Microsoft.Compute/virtualMachines/f2p5c0p1f1c4g7r-np-delete-main-gpxxw-w6pqj to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9974912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b1e08d2d-8e5a-4d42-9ec3-fa6f0e7ff42e
            correlation ID: 2fff4344-7e6c-43c1-b953-d8924fffcf5d
            client request ID: 27cbaf58-6dac-4a2b-a04f-a25864687397

alert fired 73 time(s) (17 unknown, 56 known)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T10:49:21Z
  Ended:       2026-10-15T10:55:23Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0312192-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf5r5hvchvvus9nki9e04f32q1jh0bh-cilium-cl", pod="cluster-network-operator-78f5f6bfcd-xbbzn", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf5r5hvchvvus9nki9e04f32q1jh0bh-cilium-cl/cluster-network-operator-78f5f6bfcd-xbbzn has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-15T10:49:21Z
  Ended:       2026-10-15T10:55:23Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0312192-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf5sijrpjboqn4k5qhf0lnn4bi3skan-agg-cluster", pod="dns-operator-7b96454df8-pjmnp", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf5sijrpjboqn4k5qhf0lnn4bi3skan-agg-cluster/dns-operator-7b96454df8-pjmnp has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-15T10:49:21Z
  Ended:       2026-10-15T10:55:25Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0312192-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf5r650b3uijpdce0voqlvuk621uq6k-t0v4t7j5v6v8y9o", pod="kube-scheduler-677c6d68b9-mpt59", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf5r650b3uijpdce0voqlvuk621uq6k-t0v4t7j5v6v8y9o/kube-scheduler-677c6d68b9-mpt59 has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Resolved
  Started:     2026-10-15T10:49:21Z
  Ended:       2026-10-15T10:55:25Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0312192-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf5r3i1j37cbqanpla16b30n2pl8df2-v20260901", pod="dns-operator-5fc6cfd545-ggjgz", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf5r3i1j37cbqanpla16b30n2pl8df2-v20260901/dns-operator-5fc6cfd545-ggjgz has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Resolved
  Started:     2026-10-15T10:49:21Z
  Ended:       2026-10-15T10:55:23Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0312192-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf5r3i1j37cbqanpla16b30n2pl8df2-v20260901", pod="csi-snapshot-controller-operator-574bb8fd7c-6w8cd", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf5r3i1j37cbqanpla16b30n2pl8df2-v20260901/csi-snapshot-controller-operator-574bb8fd7c-6w8cd has been in a non-ready state for longer than 5 minutes.
Firing 6:
  State:       Resolved
  Started:     2026-10-15T10:49:21Z
  Ended:       2026-10-15T10:55:24Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0312192-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf5r5ra5r546vgigjqnddshluco5df6-r3q3d8b5n5o1i4o", pod="ingress-operator-5c4c685445-ghht4", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf5r5ra5r546vgigjqnddshluco5df6-r3q3d8b5n5o1i4o/ingress-operator-5c4c685445-ghht4 has been in a non-ready state for longer than 5 minutes.
Firing 7:
  State:       Resolved
  Started:     2026-10-15T10:49:21Z
  Ended:       2026-10-15T10:55:24Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0312192-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf5r650b3uijpdce0voqlvuk621uq6k-t0v4t7j5v6v8y9o", pod="kube-controller-manager-5fbcfd4467-xwwd9", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf5r650b3uijpdce0voqlvuk621uq6k-t0v4t7j5v6v8y9o/kube-controller-manager-5fbcfd4467-xwwd9 has been in a non-ready state for longer than 5 minutes.
Firing 8:
  State:       Resolved
  Started:     2026-10-15T10:49:22Z
  Ended:       2026-10-15T10:55:25Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0312192-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf5r3i1j37cbqanpla16b30n2pl8df2-v20260901", pod="cluster-network-operator-86c7cbf5b5-5pqzd", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf5r3i1j37cbqanpla16b30n2pl8df2-v20260901/cluster-network-operator-86c7cbf5b5-5pqzd has been in a non-ready state for longer than 5 minutes.
Firing 9:
  State:       Resolved
  Started:     2026-10-15T10:49:22Z
  Ended:       2026-10-15T10:55:25Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0312192-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf5r3i1j37cbqanpla16b30n2pl8df2-v20260901", pod="ingress-operator-7976455d4c-w9m54", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf5r3i1j37cbqanpla16b30n2pl8df2-v20260901/ingress-operator-7976455d4c-w9m54 has been in a non-ready state for longer than 5 minutes.
Firing 10:
  State:       Resolved
  Started:     2026-10-15T10:49:22Z
  Ended:       2026-10-15T10:55:25Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0312192-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf5r7cddlscne6114jvj8rp9ggclk3d-private-ingress", pod="ingress-operator-7f8d7987b8-vfqw4", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf5r7cddlscne6114jvj8rp9ggclk3d-private-ingress/ingress-operator-7f8d7987b8-vfqw4 has been in a non-ready state for longer than 5 minutes.
Firing 11:
  State:       Resolved
  Started:     2026-10-15T10:49:22Z
  Ended:       2026-10-15T10:55:23Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0312192-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf5r9loa85mqspq4q6ajgf2mrm8gv04-ea-listdfcbkx", pod="ingress-operator-c55cc8c8b-pbdhs", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf5r9loa85mqspq4q6ajgf2mrm8gv04-ea-listdfcbkx/ingress-operator-c55cc8c8b-pbdhs has been in a non-ready state for longer than 5 minutes.
Firing 12:
  State:       Resolved
  Started:     2026-10-15T10:49:22Z
  Ended:       2026-10-15T10:55:22Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0312192-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf5rbm8vib0d9ebticrbnmqj531gec1-ea-cluster", pod="cluster-network-operator-6d969df6c4-vrv4t", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf5rbm8vib0d9ebticrbnmqj531gec1-ea-cluster/cluster-network-operator-6d969df6c4-vrv4t has been in a non-ready state for longer than 5 minutes.
Firing 13:
  State:       Resolved
  Started:     2026-10-15T10:49:22Z
  Ended:       2026-10-15T10:55:22Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0312192-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf5r7cddlscne6114jvj8rp9ggclk3d-private-ingress", pod="cluster-network-operator-595855fb7f-hsrm7", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf5r7cddlscne6114jvj8rp9ggclk3d-private-ingress/cluster-network-operator-595855fb7f-hsrm7 has been in a non-ready state for longer than 5 minutes.
Firing 14:
  State:       Resolved
  Started:     2026-10-15T10:49:22Z
  Ended:       2026-10-15T10:55:24Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0312192-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf5r718n6f9kpcekd9u9lvggm1tj8ci-y7w6t3k9f4c6l0j", pod="cluster-version-operator-7bf485bf7b-knn6k", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf5r718n6f9kpcekd9u9lvggm1tj8ci-y7w6t3k9f4c6l0j/cluster-version-operator-7bf485bf7b-knn6k has been in a non-ready state for longer than 5 minutes.
Firing 15:
  State:       Resolved
  Started:     2026-10-15T10:49:22Z
  Ended:       2026-10-15T10:55:24Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0312192-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf5s1hmoqh14fb514u75cpkjgslq4v4-u9f1x4e9l3o9e5u", pod="machine-approver-5c85b864d6-cq2p7", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf5s1hmoqh14fb514u75cpkjgslq4v4-u9f1x4e9l3o9e5u/machine-approver-5c85b864d6-cq2p7 has been in a non-ready state for longer than 5 minutes.
Firing 16:
  State:       Resolved
  Started:     2026-10-15T10:49:22Z
  Ended:       2026-10-15T10:55:25Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0312192-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf5r487b78dck27v37c0s6o2174bn21-q4w9i3u2i9q0h5c", pod="ingress-operator-5f965b797-krzfh", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf5r487b78dck27v37c0s6o2174bn21-q4w9i3u2i9q0h5c/ingress-operator-5f965b797-krzfh has been in a non-ready state for longer than 5 minutes.
Firing 17:
  State:       Resolved
  Started:     2026-10-15T10:49:22Z
  Ended:       2026-10-15T10:55:22Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0312192-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf5r3i1j37cbqanpla16b30n2pl8df2-v20260901", pod="cluster-storage-operator-56495bf6-t6cnq", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf5r3i1j37cbqanpla16b30n2pl8df2-v20260901/cluster-storage-operator-56495bf6-t6cnq has been in a non-ready state for longer than 5 minutes.

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T11:00:07Z
  Ended:       2026-10-15T11:20:06Z
  Severity:    Sev2
  Labels:      alertname="PrometheusUptime", cluster="ci01-j0312192-mgmt-2", component="monitoring-infrastructure", job="prometheus/prometheus", namespace="prometheus", region="centralus", severity="critical"
  Description: Fewer than 95% of present Prometheus self-up samples were healthy over the past 24 hours.
This may indicate that the Prometheus server is down, experiencing network issues, or stuck in a crash loop.
Missing samples are covered separately by PrometheusUptimeSampleCount.
Please check the status of the Prometheus pods, service endpoints, and network connectivity.

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster
Unexpected error:
    <*fmt.wrapError | 0x2a0ade6c5620>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "pullsecret-hcp-cluster" in resourcegroup="pullsecret-test-86m8pknzvtzq" to finish creating: GET https://rp.j9172992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3212538e-7b9a-48e8-b6ef-37c6f83b15a9
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3212538e-7b9a-48e8-b6ef-37c6f83b15a9",
      "name": "3212538e-7b9a-48e8-b6ef-37c6f83b15a9",
      "status": "Failed",
      "startTime": "2026-10-15T10:46:27.051179467Z",
      "endTime": "2026-10-15T11:15:31.641718353Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9172992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3212538e-7b9a-48e8-b6ef-37c6f83b15a9
            correlation ID: b2ea167c-1bb9-4987-81c1-8b8c19e034ab
            client request ID: 25be69e1-e4ac-4c05-ac28-453991fe2361
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster
Unexpected error:
    <*fmt.wrapError | 0x2a0ade6c5620>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "pullsecret-hcp-cluster" in resourcegroup="pullsecret-test-86m8pknzvtzq" to finish creating: GET https://rp.j9172992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3212538e-7b9a-48e8-b6ef-37c6f83b15a9
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3212538e-7b9a-48e8-b6ef-37c6f83b15a9",
      "name": "3212538e-7b9a-48e8-b6ef-37c6f83b15a9",
      "status": "Failed",
      "startTime": "2026-10-15T10:46:27.051179467Z",
      "endTime": "2026-10-15T11:15:31.641718353Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9172992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3212538e-7b9a-48e8-b6ef-37c6f83b15a9
            correlation ID: b2ea167c-1bb9-4987-81c1-8b8c19e034ab
            client request ID: 25be69e1-e4ac-4c05-ac28-453991fe2361
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:153]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0xa4c5bbba3a0>: 
    failed to create HCP cluster ingress-cert-le: failed waiting for cluster="ingress-cert-le" in resourcegroup="ingress-cert-le-ncdz4rpwhpzs" to finish creating: GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6f0c25df-507c-43a5-9e70-f4d938dfbf97
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6f0c25df-507c-43a5-9e70-f4d938dfbf97",
      "name": "6f0c25df-507c-43a5-9e70-f4d938dfbf97",
      "status": "Failed",
      "startTime": "2026-10-15T09:39:48.113595811Z",
      "endTime": "2026-10-15T10:08:57.753584202Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6f0c25df-507c-43a5-9e70-f4d938dfbf97
            correlation ID: 1549d38d-2ef1-4590-9c6b-a3ec1a3cc2f4
            client request ID: 5ab79f5f-ecc7-4fb3-b169-4f8f4ccf13a5
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:153]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0xa4c5bbba3a0>: 
    failed to create HCP cluster ingress-cert-le: failed waiting for cluster="ingress-cert-le" in resourcegroup="ingress-cert-le-ncdz4rpwhpzs" to finish creating: GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6f0c25df-507c-43a5-9e70-f4d938dfbf97
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6f0c25df-507c-43a5-9e70-f4d938dfbf97",
      "name": "6f0c25df-507c-43a5-9e70-f4d938dfbf97",
      "status": "Failed",
      "startTime": "2026-10-15T09:39:48.113595811Z",
      "endTime": "2026-10-15T10:08:57.753584202Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6f0c25df-507c-43a5-9e70-f4d938dfbf97
            correlation ID: 1549d38d-2ef1-4590-9c6b-a3ec1a3cc2f4
            client request ID: 5ab79f5f-ecc7-4fb3-b169-4f8f4ccf13a5
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-20-7xw44w" with version 4.20.41 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x1e014d2781c0>: 
    failed to create HCP cluster cluster-zstream-4-20-7xw44w: failed waiting for cluster="cluster-zstream-4-20-7xw44w" in resourcegroup="rg-zstream-upgrade-4-20-m9dzg9b4mkpj" to finish creating: GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/40a40396-7b07-4032-84dc-85483287fa77
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/40a40396-7b07-4032-84dc-85483287fa77",
      "name": "40a40396-7b07-4032-84dc-85483287fa77",
      "status": "Failed",
      "startTime": "2026-10-15T09:39:48.143600215Z",
      "endTime": "2026-10-15T10:08:57.782039441Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/40a40396-7b07-4032-84dc-85483287fa77
            correlation ID: c1f76f8b-0244-4f84-9b33-78a9ec6f02fc
            client request ID: 435434cd-57a6-4a90-8709-e50f7498118a
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-20-7xw44w" with version 4.20.41 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x1e014d2781c0>: 
    failed to create HCP cluster cluster-zstream-4-20-7xw44w: failed waiting for cluster="cluster-zstream-4-20-7xw44w" in resourcegroup="rg-zstream-upgrade-4-20-m9dzg9b4mkpj" to finish creating: GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/40a40396-7b07-4032-84dc-85483287fa77
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/40a40396-7b07-4032-84dc-85483287fa77",
      "name": "40a40396-7b07-4032-84dc-85483287fa77",
      "status": "Failed",
      "startTime": "2026-10-15T09:39:48.143600215Z",
      "endTime": "2026-10-15T10:08:57.782039441Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/40a40396-7b07-4032-84dc-85483287fa77
            correlation ID: c1f76f8b-0244-4f84-9b33-78a9ec6f02fc
            client request ID: 435434cd-57a6-4a90-8709-e50f7498118a
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-21-fzffs7" with version 4.21.36 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x2f680e6086a0>: 
    failed to create HCP cluster cluster-zstream-4-21-fzffs7: failed waiting for cluster="cluster-zstream-4-21-fzffs7" in resourcegroup="rg-zstream-upgrade-4-21-fk78wfgpv6f4" to finish creating: GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f992823-ace9-4bbf-93da-fc0443bfd8c7
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f992823-ace9-4bbf-93da-fc0443bfd8c7",
      "name": "4f992823-ace9-4bbf-93da-fc0443bfd8c7",
      "status": "Failed",
      "startTime": "2026-10-15T09:40:08.845848533Z",
      "endTime": "2026-10-15T10:09:17.834288486Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f992823-ace9-4bbf-93da-fc0443bfd8c7
            correlation ID: 3da74456-40a1-495e-a2ee-ae1aa1e064b2
            client request ID: 063711c3-7c8b-4258-ac53-1eeaec215430
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-21-fzffs7" with version 4.21.36 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x2f680e6086a0>: 
    failed to create HCP cluster cluster-zstream-4-21-fzffs7: failed waiting for cluster="cluster-zstream-4-21-fzffs7" in resourcegroup="rg-zstream-upgrade-4-21-fk78wfgpv6f4" to finish creating: GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f992823-ace9-4bbf-93da-fc0443bfd8c7
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f992823-ace9-4bbf-93da-fc0443bfd8c7",
      "name": "4f992823-ace9-4bbf-93da-fc0443bfd8c7",
      "status": "Failed",
      "startTime": "2026-10-15T09:40:08.845848533Z",
      "endTime": "2026-10-15T10:09:17.834288486Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f992823-ace9-4bbf-93da-fc0443bfd8c7
            correlation ID: 3da74456-40a1-495e-a2ee-ae1aa1e064b2
            client request ID: 063711c3-7c8b-4258-ac53-1eeaec215430
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:114]: failed to create cluster "sre-version-pin-k97lng" at 4.21.36
Unexpected error:
    <*fmt.wrapError | 0x1533100288e0>: 
    failed to create HCP cluster sre-version-pin-k97lng: failed waiting for cluster="sre-version-pin-k97lng" in resourcegroup="admin-api-version-pin-srtm7jx8t2zb" to finish creating: GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b84e5174-a51f-4ccb-81ff-7fe8387b9ff7
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b84e5174-a51f-4ccb-81ff-7fe8387b9ff7",
      "name": "b84e5174-a51f-4ccb-81ff-7fe8387b9ff7",
      "status": "Failed",
      "startTime": "2026-10-15T09:40:00.953675448Z",
      "endTime": "2026-10-15T10:09:07.839497723Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b84e5174-a51f-4ccb-81ff-7fe8387b9ff7
            correlation ID: ee96e857-47ed-4d45-8af0-6c84276e732e
            client request ID: 54710fb8-fb6a-4b6b-b646-27d3dc3d8364
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:114]: failed to create cluster "sre-version-pin-k97lng" at 4.21.36
Unexpected error:
    <*fmt.wrapError | 0x1533100288e0>: 
    failed to create HCP cluster sre-version-pin-k97lng: failed waiting for cluster="sre-version-pin-k97lng" in resourcegroup="admin-api-version-pin-srtm7jx8t2zb" to finish creating: GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b84e5174-a51f-4ccb-81ff-7fe8387b9ff7
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b84e5174-a51f-4ccb-81ff-7fe8387b9ff7",
      "name": "b84e5174-a51f-4ccb-81ff-7fe8387b9ff7",
      "status": "Failed",
      "startTime": "2026-10-15T09:40:00.953675448Z",
      "endTime": "2026-10-15T10:09:07.839497723Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b84e5174-a51f-4ccb-81ff-7fe8387b9ff7
            correlation ID: ee96e857-47ed-4d45-8af0-6c84276e732e
            client request ID: 54710fb8-fb6a-4b6b-b646-27d3dc3d8364
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:153]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0x2a68ee7e6c80>: 
    failed to create HCP cluster ingress-cert-le: failed waiting for cluster="ingress-cert-le" in resourcegroup="ingress-cert-le-sb29tw87pzgh" to finish creating: GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/86a0f5fb-3099-4294-9dc3-bf5f35c7d457
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/86a0f5fb-3099-4294-9dc3-bf5f35c7d457",
      "name": "86a0f5fb-3099-4294-9dc3-bf5f35c7d457",
      "status": "Failed",
      "startTime": "2026-10-15T09:39:47.815555274Z",
      "endTime": "2026-10-15T10:08:47.869197082Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/86a0f5fb-3099-4294-9dc3-bf5f35c7d457
            correlation ID: f1a60f2c-b997-42cd-8990-f50df17163bd
            client request ID: b5612464-42d4-40fe-8f75-9d5b4945da98
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:153]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0x2a68ee7e6c80>: 
    failed to create HCP cluster ingress-cert-le: failed waiting for cluster="ingress-cert-le" in resourcegroup="ingress-cert-le-sb29tw87pzgh" to finish creating: GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/86a0f5fb-3099-4294-9dc3-bf5f35c7d457
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/86a0f5fb-3099-4294-9dc3-bf5f35c7d457",
      "name": "86a0f5fb-3099-4294-9dc3-bf5f35c7d457",
      "status": "Failed",
      "startTime": "2026-10-15T09:39:47.815555274Z",
      "endTime": "2026-10-15T10:08:47.869197082Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/86a0f5fb-3099-4294-9dc3-bf5f35c7d457
            correlation ID: f1a60f2c-b997-42cd-8990-f50df17163bd
            client request ID: b5612464-42d4-40fe-8f75-9d5b4945da98
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20
Unexpected error:
    <*fmt.wrapErrors | 0x3e6f09e44c00>: 
    failed waiting for cluster="cluster-ver-4-20" in resourcegroup="rg-cluster-back-version-k7gbzwzskr9p" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-k7gbzwzskr9p, error: context deadline exceeded
    {
        msg: "failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-k7gbzwzskr9p\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-k7gbzwzskr9p, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0x3e6f0a152150>{
                s: "timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-k7gbzwzskr9p",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20
Unexpected error:
    <*fmt.wrapErrors | 0x3e6f09e44c00>: 
    failed waiting for cluster="cluster-ver-4-20" in resourcegroup="rg-cluster-back-version-k7gbzwzskr9p" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-k7gbzwzskr9p, error: context deadline exceeded
    {
        msg: "failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-k7gbzwzskr9p\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-k7gbzwzskr9p, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0x3e6f0a152150>{
                s: "timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-k7gbzwzskr9p",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-knrjczrxgllf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1813dab12220>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-knrjczrxgllf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-knrjczrxgllf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T09:24:23.3470894Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-knrjczrxgllf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-knrjczrxgllf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T09:24:23.3470894Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1813dab12200>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-knrjczrxgllf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T09:24:23.3470894Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1813dae08140>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1813dab121e0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-knrjczrxgllf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T09:24:23.3470894Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1813dc73c0c0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-knrjczrxgllf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1813dab12220>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-knrjczrxgllf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-knrjczrxgllf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T09:24:23.3470894Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-knrjczrxgllf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-knrjczrxgllf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T09:24:23.3470894Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1813dab12200>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-knrjczrxgllf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T09:24:23.3470894Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1813dae08140>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1813dab121e0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-knrjczrxgllf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T09:24:23.3470894Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1813dc73c0c0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster "cilium-cl" without CNI
Unexpected error:
    <*fmt.wrapError | 0x1cdd9060dca0>: 
    failed to create HCP cluster cilium-cl: failed waiting for cluster="cilium-cl" in resourcegroup="cni-cilium-lqb44pqrvnjn" to finish creating: GET https://rp.j3413376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1eb1b48b-885e-44b5-a938-88689e5212e6
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: ResourceNotFound
    --------------------------------------------------------------------------------
    {
      "error": {
        "code": "ResourceNotFound",
        "message": "The resource 'locations/hcpOperationStatuses/1eb1b48b-885e-44b5-a938-88689e5212e6' under resource group '' was not found.",
        "target": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1eb1b48b-885e-44b5-a938-88689e5212e6"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3413376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1eb1b48b-885e-44b5-a938-88689e5212e6
            correlation ID: 70fa4741-a1d2-4d88-b00c-c2caa1e45683
            client request ID: c17bea04-7230-45d2-bf8b-2cdbaefe70c1
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster "cilium-cl" without CNI
Unexpected error:
    <*fmt.wrapError | 0x1cdd9060dca0>: 
    failed to create HCP cluster cilium-cl: failed waiting for cluster="cilium-cl" in resourcegroup="cni-cilium-lqb44pqrvnjn" to finish creating: GET https://rp.j3413376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1eb1b48b-885e-44b5-a938-88689e5212e6
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: ResourceNotFound
    --------------------------------------------------------------------------------
    {
      "error": {
        "code": "ResourceNotFound",
        "message": "The resource 'locations/hcpOperationStatuses/1eb1b48b-885e-44b5-a938-88689e5212e6' under resource group '' was not found.",
        "target": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1eb1b48b-885e-44b5-a938-88689e5212e6"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3413376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1eb1b48b-885e-44b5-a938-88689e5212e6
            correlation ID: 70fa4741-a1d2-4d88-b00c-c2caa1e45683
            client request ID: c17bea04-7230-45d2-bf8b-2cdbaefe70c1
occurred

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T09:51:08Z
  Ended:       2026-10-15T10:01:06Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j3413376-svc", component="backend", name="clusterresources", region="westus3", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T10:40:31Z
  Severity:    Sev3
  Labels:      alertname="LeaderElectionLeaseStale", cluster="ci01-j3413376-mgmt-1", component="leader-election", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.65.38:8080", job="kube-state-metrics", lease="work-manager", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3413376/providers/microsoft.monitor/accounts/services-j3413376", namespace="klusterlet-2tf50j114ec16t9gt17n7jkkno5ih42t", pod="arohcp-monitor-kube-state-metrics-7988dc648c-m98dk", prometheus="prometheus/prometheus", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Leader election lease work-manager in namespace klusterlet-2tf50j114ec16t9gt17n7jkkno5ih42t on cluster ci01-j3413376-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-fs95n68z9x97/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-fs95n68z9x97/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:293]: failed to create cluster customer resources
Unexpected error:
    <*fmt.wrapError | 0x2d44881448c0>: 
    failed to create managed identities: failed to create managed identities: failed waiting for deployment "mi-np-noedge-m2mjbc-x5mdss" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-m2mjbc-x5mdss
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: DeploymentNotFound
    --------------------------------------------------------------------------------
    {
      "error": {
        "code": "DeploymentNotFound",
        "message": "Deployment 'mi-np-noedge-m2mjbc-x5mdss' could not be found."
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-m2mjbc-x5mdss
            correlation ID: fd414cb6-2644-4527-9130-ea3d146c0317
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:293]: failed to create cluster customer resources
Unexpected error:
    <*fmt.wrapError | 0x2d44881448c0>: 
    failed to create managed identities: failed to create managed identities: failed waiting for deployment "mi-np-noedge-m2mjbc-x5mdss" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-m2mjbc-x5mdss
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: DeploymentNotFound
    --------------------------------------------------------------------------------
    {
      "error": {
        "code": "DeploymentNotFound",
        "message": "Deployment 'mi-np-noedge-m2mjbc-x5mdss' could not be found."
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-m2mjbc-x5mdss
            correlation ID: fd414cb6-2644-4527-9130-ea3d146c0317
            client request ID: unavailable
occurred

alert fired 34 time(s) (28 unknown, 6 known)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-6qp87", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-6qp87 has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-592vw", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-592vw has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:48Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-42rw7", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-42rw7 has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-wj9q5", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-wj9q5 has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-xg7tz", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-xg7tz has been in a non-ready state for longer than 5 minutes.
Firing 6:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-r24fs", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-r24fs has been in a non-ready state for longer than 5 minutes.
Firing 7:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-qgcnj", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-qgcnj has been in a non-ready state for longer than 5 minutes.
Firing 8:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:48Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-jl2qm", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-jl2qm has been in a non-ready state for longer than 5 minutes.
Firing 9:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-tthdg", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-tthdg has been in a non-ready state for longer than 5 minutes.
Firing 10:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-4cqst", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-4cqst has been in a non-ready state for longer than 5 minutes.
Firing 11:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-96qpz", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-96qpz has been in a non-ready state for longer than 5 minutes.
Firing 12:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-b26k9", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-b26k9 has been in a non-ready state for longer than 5 minutes.
Firing 13:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-wlxt4", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-wlxt4 has been in a non-ready state for longer than 5 minutes.
Firing 14:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:48Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-9g95b", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-9g95b has been in a non-ready state for longer than 5 minutes.
Firing 15:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-qx86v", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-qx86v has been in a non-ready state for longer than 5 minutes.
Firing 16:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:48Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-znp49", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-znp49 has been in a non-ready state for longer than 5 minutes.
Firing 17:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:48Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-n6sw2", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-n6sw2 has been in a non-ready state for longer than 5 minutes.
Firing 18:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-4b8mt", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-4b8mt has been in a non-ready state for longer than 5 minutes.
Firing 19:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-bs72h", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-bs72h has been in a non-ready state for longer than 5 minutes.
Firing 20:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-qxb6s", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-qxb6s has been in a non-ready state for longer than 5 minutes.
Firing 21:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-95hwp", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-95hwp has been in a non-ready state for longer than 5 minutes.
Firing 22:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:48Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-d5nxq", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-d5nxq has been in a non-ready state for longer than 5 minutes.
Firing 23:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-fp64c", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-fp64c has been in a non-ready state for longer than 5 minutes.
Firing 24:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-cfb4c", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-cfb4c has been in a non-ready state for longer than 5 minutes.
Firing 25:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-wqqrk", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-wqqrk has been in a non-ready state for longer than 5 minutes.
Firing 26:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:48Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-6kbcx", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-6kbcx has been in a non-ready state for longer than 5 minutes.
Firing 27:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-sphw9", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-sphw9 has been in a non-ready state for longer than 5 minutes.
Firing 28:
  State:       Resolved
  Started:     2026-10-15T09:03:51Z
  Ended:       2026-10-15T09:23:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9515776-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f", pod="network-node-identity-64564cfc95-5clxc", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf460dguuc9566ac92ujr39nl0t9oj1-f9z8f7r6y7a2p3f/network-node-identity-64564cfc95-5clxc has been in a non-ready state for longer than 5 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:160]: failed to create node pool np-1
Unexpected error:
    <*fmt.wrapError | 0x3905b4102dc0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "negative-tests-cluster" in resourcegroup="rg-negative-tests-6hvwcqt9kdnj" to finish creating: GET https://rp.j2254592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9969c6be-dd3e-417c-a579-9c56fbcbd085
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9969c6be-dd3e-417c-a579-9c56fbcbd085",
      "name": "9969c6be-dd3e-417c-a579-9c56fbcbd085",
      "status": "Failed",
      "startTime": "2026-10-15T09:01:01.486597896Z",
      "endTime": "2026-10-15T09:30:03.363245379Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine u2o1n0b9r6v5w2p-np-1-gdbl2-lhb8q: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-6hvwcqt9kdnj--managed/providers/Microsoft.Compute/virtualMachines/u2o1n0b9r6v5w2p-np-1-gdbl2-lhb8q to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine u2o1n0b9r6v5w2p-np-1-gdbl2-lhb8q: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-6hvwcqt9kdnj--managed/providers/Microsoft.Compute/virtualMachines/u2o1n0b9r6v5w2p-np-1-gdbl2-lhb8q to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2254592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9969c6be-dd3e-417c-a579-9c56fbcbd085
            correlation ID: ec950d26-97a5-4fe2-ae63-dad821b3f8a4
            client request ID: 80a098fb-d4e0-45cd-ac80-281df3707c8c
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:160]: failed to create node pool np-1
Unexpected error:
    <*fmt.wrapError | 0x3905b4102dc0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "negative-tests-cluster" in resourcegroup="rg-negative-tests-6hvwcqt9kdnj" to finish creating: GET https://rp.j2254592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9969c6be-dd3e-417c-a579-9c56fbcbd085
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9969c6be-dd3e-417c-a579-9c56fbcbd085",
      "name": "9969c6be-dd3e-417c-a579-9c56fbcbd085",
      "status": "Failed",
      "startTime": "2026-10-15T09:01:01.486597896Z",
      "endTime": "2026-10-15T09:30:03.363245379Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine u2o1n0b9r6v5w2p-np-1-gdbl2-lhb8q: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-6hvwcqt9kdnj--managed/providers/Microsoft.Compute/virtualMachines/u2o1n0b9r6v5w2p-np-1-gdbl2-lhb8q to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine u2o1n0b9r6v5w2p-np-1-gdbl2-lhb8q: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-6hvwcqt9kdnj--managed/providers/Microsoft.Compute/virtualMachines/u2o1n0b9r6v5w2p-np-1-gdbl2-lhb8q to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2254592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9969c6be-dd3e-417c-a579-9c56fbcbd085
            correlation ID: ec950d26-97a5-4fe2-ae63-dad821b3f8a4
            client request ID: 80a098fb-d4e0-45cd-ac80-281df3707c8c
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:365]: Timed out after 300.006s.
ClusterIssuer should become Ready (ACME account registered)
Expected success, but got an error:
    <*errors.errorString | 0x3f466db88d40>: 
    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get "https://acme-staging-v02.api.letsencrypt.org/directory": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:33148->172.30.0.10:53: read: connection refused
    {
        s: "clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:33148->172.30.0.10:53: read: connection refused",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:365]: Timed out after 300.006s.
ClusterIssuer should become Ready (ACME account registered)
Expected success, but got an error:
    <*errors.errorString | 0x3f466db88d40>: 
    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get "https://acme-staging-v02.api.letsencrypt.org/directory": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:33148->172.30.0.10:53: read: connection refused
    {
        s: "clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:33148->172.30.0.10:53: read: connection refused",
    }

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T08:12:35Z
  Ended:       2026-10-15T08:33:35Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j2847232-svc", component="backend", name="actualhostedcluster", region="centralus", severity="warning"
  Description: Backend controller workqueue actualhostedcluster has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T08:53:17Z
  Severity:    Sev3
  Labels:      alertname="LeaderElectionLeaseStale", cluster="ci01-j2847232-mgmt-1", component="leader-election", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.97:8080", job="kube-state-metrics", lease="work-manager", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2847232/providers/microsoft.monitor/accounts/services-j2847232", namespace="klusterlet-2tf3h722shfll7pqd2s7sbjc21ck4oqd", pod="arohcp-monitor-kube-state-metrics-7988dc648c-7k2fb", prometheus="prometheus/prometheus", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Leader election lease work-manager in namespace klusterlet-2tf3h722shfll7pqd2s7sbjc21ck4oqd on cluster ci01-j2847232-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.

alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T08:52:44Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2847232-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.90:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2847232/providers/microsoft.monitor/accounts/hcps-j2847232", namespace="ocm-arohcpci01-2tf3heagkirok77h5j6fppm92slv02n8-private-ingress", pod="arohcp-monitor-kube-state-metrics-7988dc648c-jp5hq", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tf3heagkirok77h5j6fppm92slv02n8-private-ingress/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-15T08:53:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2847232-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.90:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2847232/providers/microsoft.monitor/accounts/hcps-j2847232", namespace="ocm-arohcpci01-2tf3heagkirok77h5j6fppm92slv02n8-private-ingress", pod="arohcp-monitor-kube-state-metrics-7988dc648c-jp5hq", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tf3heagkirok77h5j6fppm92slv02n8-private-ingress/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool "np-xr525c" for cluster "cluster-hs-pre-xr525c"
Unexpected error:
    <*fmt.wrapError | 0x29035dae21e0>: 
    failed to create NodePool np-xr525c: failed waiting for nodepool="np-xr525c" for cluster "cluster-hs-pre-xr525c" in resourcegroup="rg-hs-pre--p4xbkh8wjm7g" to finish creating: GET https://rp.j5169664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d9a16139-7271-459b-8789-dac587c14c34
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d9a16139-7271-459b-8789-dac587c14c34",
      "name": "d9a16139-7271-459b-8789-dac587c14c34",
      "status": "Failed",
      "startTime": "2026-10-15T07:15:59.185365451Z",
      "endTime": "2026-10-15T07:45:03.436202817Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine r0q0q8c3o2w1s9a-np-xr525c-s7h72-55bwn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--p4xbkh8wjm7g-xr525c--managed/providers/Microsoft.Compute/virtualMachines/r0q0q8c3o2w1s9a-np-xr525c-s7h72-55bwn to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine r0q0q8c3o2w1s9a-np-xr525c-s7h72-55bwn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--p4xbkh8wjm7g-xr525c--managed/providers/Microsoft.Compute/virtualMachines/r0q0q8c3o2w1s9a-np-xr525c-s7h72-55bwn to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5169664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d9a16139-7271-459b-8789-dac587c14c34
            correlation ID: 3020ad54-6f72-4e03-b73c-90532cc6f08d
            client request ID: 41bb47ed-7e61-49ca-8e9c-daebc3ef414b
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool "np-xr525c" for cluster "cluster-hs-pre-xr525c"
Unexpected error:
    <*fmt.wrapError | 0x29035dae21e0>: 
    failed to create NodePool np-xr525c: failed waiting for nodepool="np-xr525c" for cluster "cluster-hs-pre-xr525c" in resourcegroup="rg-hs-pre--p4xbkh8wjm7g" to finish creating: GET https://rp.j5169664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d9a16139-7271-459b-8789-dac587c14c34
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d9a16139-7271-459b-8789-dac587c14c34",
      "name": "d9a16139-7271-459b-8789-dac587c14c34",
      "status": "Failed",
      "startTime": "2026-10-15T07:15:59.185365451Z",
      "endTime": "2026-10-15T07:45:03.436202817Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine r0q0q8c3o2w1s9a-np-xr525c-s7h72-55bwn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--p4xbkh8wjm7g-xr525c--managed/providers/Microsoft.Compute/virtualMachines/r0q0q8c3o2w1s9a-np-xr525c-s7h72-55bwn to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine r0q0q8c3o2w1s9a-np-xr525c-s7h72-55bwn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--p4xbkh8wjm7g-xr525c--managed/providers/Microsoft.Compute/virtualMachines/r0q0q8c3o2w1s9a-np-xr525c-s7h72-55bwn to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5169664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d9a16139-7271-459b-8789-dac587c14c34
            correlation ID: 3020ad54-6f72-4e03-b73c-90532cc6f08d
            client request ID: 41bb47ed-7e61-49ca-8e9c-daebc3ef414b
occurred

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:
    <*errors.joinError | 0x1808f1276588>: 
    resource group "des-encrypt-fr8cqml7zsjj": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="des-encrypt" in resourcegroup="des-encrypt-fr8cqml7zsjj" to finish deleting: GET https://rp.j7707648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d46ca823-6a99-453f-a91b-c390aed9f889
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d46ca823-6a99-453f-a91b-c390aed9f889",
      "name": "d46ca823-6a99-453f-a91b-c390aed9f889",
      "status": "Failed",
      "startTime": "2026-10-15T06:45:35.515831022Z",
      "endTime": "2026-10-15T07:09:37.554161836Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tf236h3grf5nh5cka2klv381nroha64 still exists (deletion dispatched at 2026-10-15T06:45:36Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7707648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d46ca823-6a99-453f-a91b-c390aed9f889
            correlation ID: 9dd61a19-eb6b-42e3-962a-725be28358e3
            client request ID: 0e89fe2d-1574-456d-9c74-1a2277b25a8d
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:
    <*errors.joinError | 0x1808f1276588>: 
    resource group "des-encrypt-fr8cqml7zsjj": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="des-encrypt" in resourcegroup="des-encrypt-fr8cqml7zsjj" to finish deleting: GET https://rp.j7707648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d46ca823-6a99-453f-a91b-c390aed9f889
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d46ca823-6a99-453f-a91b-c390aed9f889",
      "name": "d46ca823-6a99-453f-a91b-c390aed9f889",
      "status": "Failed",
      "startTime": "2026-10-15T06:45:35.515831022Z",
      "endTime": "2026-10-15T07:09:37.554161836Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tf236h3grf5nh5cka2klv381nroha64 still exists (deletion dispatched at 2026-10-15T06:45:36Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7707648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d46ca823-6a99-453f-a91b-c390aed9f889
            correlation ID: 9dd61a19-eb6b-42e3-962a-725be28358e3
            client request ID: 0e89fe2d-1574-456d-9c74-1a2277b25a8d
occurred

alert fired 4 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T07:28:07Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7707648-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cluster-api", endpoint="http", environment="ci01", instance="10.128.64.196:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7707648/providers/microsoft.monitor/accounts/hcps-j7707648", namespace="ocm-arohcpci01-2tf236h3grf5nh5cka2klv381nroha64-des-encrypt", pod="arohcp-monitor-kube-state-metrics-7988dc648c-wd85h", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tf236h3grf5nh5cka2klv381nroha64-des-encrypt/cluster-api has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-15T07:28:07Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7707648-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.196:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7707648/providers/microsoft.monitor/accounts/hcps-j7707648", namespace="ocm-arohcpci01-2tf236h3grf5nh5cka2klv381nroha64-des-encrypt", pod="arohcp-monitor-kube-state-metrics-7988dc648c-wd85h", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tf236h3grf5nh5cka2klv381nroha64-des-encrypt/capi-provider has not matched the expected number of replicas for longer than 30 minutes.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-15T07:28:07Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7707648-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cluster-api", endpoint="http", environment="ci01", instance="10.128.64.196:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7707648/providers/microsoft.monitor/accounts/hcps-j7707648", namespace="ocm-arohcpci01-2tf236h3grf5nh5cka2klv381nroha64-des-encrypt", pod="arohcp-monitor-kube-state-metrics-7988dc648c-wd85h", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tf236h3grf5nh5cka2klv381nroha64-des-encrypt/cluster-api has not matched the expected number of replicas for longer than 30 minutes.
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-15T07:28:07Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7707648-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.196:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7707648/providers/microsoft.monitor/accounts/hcps-j7707648", namespace="ocm-arohcpci01-2tf236h3grf5nh5cka2klv381nroha64-des-encrypt", pod="arohcp-monitor-kube-state-metrics-7988dc648c-wd85h", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tf236h3grf5nh5cka2klv381nroha64-des-encrypt/capi-provider has not matched the expected number of replicas for longer than 30 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI
Unexpected error:
    <*errors.joinError | 0x18c85c0bedc8>: 
    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded
    {
        errs: [
            <*fmt.wrapError | 0x18c85ad32dc0>{
                msg: "VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
                err: <*fmt.wrapError | 0x18c85bf9a260>{
                    msg: "DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
                    err: <*fmt.wrapError | 0x18c85bf9a240>{
                        msg: "DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
                        err: <context.deadlineExceededError>{},
                    },
                },
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI
Unexpected error:
    <*errors.joinError | 0x18c85c0bedc8>: 
    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded
    {
        errs: [
            <*fmt.wrapError | 0x18c85ad32dc0>{
                msg: "VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
                err: <*fmt.wrapError | 0x18c85bf9a260>{
                    msg: "DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
                    err: <*fmt.wrapError | 0x18c85bf9a240>{
                        msg: "DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
                        err: <context.deadlineExceededError>{},
                    },
                },
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x38b9840538e0>: 
    failed to create NodePool np-delete-main: failed waiting for nodepool="np-delete-main" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-8w5l7nl2mg2p" to finish creating: GET https://rp.j7178624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f20f693c-30b3-4028-bef9-79a61131ae05
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f20f693c-30b3-4028-bef9-79a61131ae05",
      "name": "f20f693c-30b3-4028-bef9-79a61131ae05",
      "status": "Failed",
      "startTime": "2026-10-15T06:40:48.977059858Z",
      "endTime": "2026-10-15T07:09:53.223490028Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine i3x0k0f4h4m3h2m-np-delete-main-97ksv-4fhzs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8w5l7nl2mg2p--managed/providers/Microsoft.Compute/virtualMachines/i3x0k0f4h4m3h2m-np-delete-main-97ksv-4fhzs to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine i3x0k0f4h4m3h2m-np-delete-main-97ksv-4fhzs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8w5l7nl2mg2p--managed/providers/Microsoft.Compute/virtualMachines/i3x0k0f4h4m3h2m-np-delete-main-97ksv-4fhzs to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7178624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f20f693c-30b3-4028-bef9-79a61131ae05
            correlation ID: c5b64ec2-f2e3-4ea2-80c9-40d2bea7a8eb
            client request ID: 8db26239-6ff9-4e91-938f-c7c27b5dc1e1
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x38b9840538e0>: 
    failed to create NodePool np-delete-main: failed waiting for nodepool="np-delete-main" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-8w5l7nl2mg2p" to finish creating: GET https://rp.j7178624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f20f693c-30b3-4028-bef9-79a61131ae05
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f20f693c-30b3-4028-bef9-79a61131ae05",
      "name": "f20f693c-30b3-4028-bef9-79a61131ae05",
      "status": "Failed",
      "startTime": "2026-10-15T06:40:48.977059858Z",
      "endTime": "2026-10-15T07:09:53.223490028Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine i3x0k0f4h4m3h2m-np-delete-main-97ksv-4fhzs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8w5l7nl2mg2p--managed/providers/Microsoft.Compute/virtualMachines/i3x0k0f4h4m3h2m-np-delete-main-97ksv-4fhzs to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine i3x0k0f4h4m3h2m-np-delete-main-97ksv-4fhzs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8w5l7nl2mg2p--managed/providers/Microsoft.Compute/virtualMachines/i3x0k0f4h4m3h2m-np-delete-main-97ksv-4fhzs to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7178624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f20f693c-30b3-4028-bef9-79a61131ae05
            correlation ID: c5b64ec2-f2e3-4ea2-80c9-40d2bea7a8eb
            client request ID: 8db26239-6ff9-4e91-938f-c7c27b5dc1e1

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1d47db71ece0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T06:26:16.8623213Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T06:26:16.8623213Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1d47db71eca0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T06:26:16.8623213Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1d47d9c628c0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1d47db71ec80>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T06:26:16.8623213Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1d47d9ddf1d0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1d47db71ece0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T06:26:16.8623213Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T06:26:16.8623213Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1d47db71eca0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T06:26:16.8623213Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1d47d9c628c0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1d47db71ec80>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T06:26:16.8623213Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1d47d9ddf1d0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w6xbc82hxr2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x280af577a000>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w6xbc82hxr2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-2tf1898kdmei2jv6trajehsfpj8o4bk2-f7r5q8n1l4e4d9o: expected all current routers >= desired replicas > 0; desired=0 current=0 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w6xbc82hxr2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-2tf1898kdmei2jv6trajehsfpj8o4bk2-f7r5q8n1l4e4d9o: expected all current routers >= desired replicas > 0; desired=0 current=0 rows=1",
        err: <*errors.errorString | 0x280af63caa80>{
            s: "dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-2tf1898kdmei2jv6trajehsfpj8o4bk2-f7r5q8n1l4e4d9o: expected all current routers >= desired replicas > 0; desired=0 current=0 rows=1",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w6xbc82hxr2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x280af577a000>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w6xbc82hxr2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-2tf1898kdmei2jv6trajehsfpj8o4bk2-f7r5q8n1l4e4d9o: expected all current routers >= desired replicas > 0; desired=0 current=0 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w6xbc82hxr2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-2tf1898kdmei2jv6trajehsfpj8o4bk2-f7r5q8n1l4e4d9o: expected all current routers >= desired replicas > 0; desired=0 current=0 rows=1",
        err: <*errors.errorString | 0x280af63caa80>{
            s: "dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-2tf1898kdmei2jv6trajehsfpj8o4bk2-f7r5q8n1l4e4d9o: expected all current routers >= desired replicas > 0; desired=0 current=0 rows=1",
        },
    }
occurred

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T05:57:23Z
  Ended:       2026-10-15T06:03:24Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8670976-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tf181j4rusq6pkjnivolrv9gud9pk5v-x9q9a7d4u8q5f3x", pod="ignition-server-6cfc56b87-qjx7r", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tf181j4rusq6pkjnivolrv9gud9pk5v-x9q9a7d4u8q5f3x/ignition-server-6cfc56b87-qjx7r has been in a non-ready state for longer than 5 minutes.

alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T06:10:13Z
  Severity:    Sev3
  Labels:      alertname="LeaderElectionLeaseStale", cluster="ci01-j8670976-mgmt-1", component="leader-election", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.103:8080", job="kube-state-metrics", lease="work-manager", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8670976/providers/microsoft.monitor/accounts/services-j8670976", namespace="klusterlet-2tf18dgs468rqeforqnek754iblq2hqg", pod="arohcp-monitor-kube-state-metrics-7988dc648c-4r5xr", prometheus="prometheus/prometheus", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Leader election lease work-manager in namespace klusterlet-2tf18dgs468rqeforqnek754iblq2hqg on cluster ci01-j8670976-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T05:33:44Z
  Ended:       2026-10-15T05:53:30Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j8670976-svc", component="backend", name="clusterresources", region="centralus", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-mpppvh with version 4.21.37
Unexpected error:
    <*fmt.wrapError | 0x1bc8b1824920>: 
    failed to create HCP cluster np-version-upgrade-cluster-mpppvh: failed starting cluster creation "np-version-upgrade-cluster-mpppvh" in resourcegroup="rg-np-version-upgrade-mpppvh-6ckrm5njjkhq": PUT https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mpppvh-6ckrm5njjkhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mpppvh
    --------------------------------------------------------------------------------
    RESPONSE 409: 409 Conflict
    ERROR CODE: Conflict
    --------------------------------------------------------------------------------
    {
      "error": {
        "code": "Conflict",
        "message": "Cannot update resource while resource is \"accepted\"",
        "target": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mpppvh-6ckrm5njjkhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mpppvh"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          PUT https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mpppvh-6ckrm5njjkhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mpppvh
            correlation ID: 5672e540-0082-400e-8d95-fd1926fa87e0
            client request ID: 32f4cab2-abf5-42a2-aa1e-6a050c192397
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-mpppvh with version 4.21.37
Unexpected error:
    <*fmt.wrapError | 0x1bc8b1824920>: 
    failed to create HCP cluster np-version-upgrade-cluster-mpppvh: failed starting cluster creation "np-version-upgrade-cluster-mpppvh" in resourcegroup="rg-np-version-upgrade-mpppvh-6ckrm5njjkhq": PUT https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mpppvh-6ckrm5njjkhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mpppvh
    --------------------------------------------------------------------------------
    RESPONSE 409: 409 Conflict
    ERROR CODE: Conflict
    --------------------------------------------------------------------------------
    {
      "error": {
        "code": "Conflict",
        "message": "Cannot update resource while resource is \"accepted\"",
        "target": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mpppvh-6ckrm5njjkhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mpppvh"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          PUT https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mpppvh-6ckrm5njjkhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mpppvh
            correlation ID: 5672e540-0082-400e-8d95-fd1926fa87e0
            client request ID: 32f4cab2-abf5-42a2-aa1e-6a050c192397
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tzcs2w52g775/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x6f640d28320>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tzcs2w52g775/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-2tf0946f9b41h1n7hltqnq95f64jvhk8-j9h5p7q3y7t6s2u: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tzcs2w52g775/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-2tf0946f9b41h1n7hltqnq95f64jvhk8-j9h5p7q3y7t6s2u: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*errors.errorString | 0x6f64075f4f0>{
            s: "dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-2tf0946f9b41h1n7hltqnq95f64jvhk8-j9h5p7q3y7t6s2u: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tzcs2w52g775/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x6f640d28320>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tzcs2w52g775/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-2tf0946f9b41h1n7hltqnq95f64jvhk8-j9h5p7q3y7t6s2u: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tzcs2w52g775/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-2tf0946f9b41h1n7hltqnq95f64jvhk8-j9h5p7q3y7t6s2u: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*errors.errorString | 0x6f64075f4f0>{
            s: "dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-2tf0946f9b41h1n7hltqnq95f64jvhk8-j9h5p7q3y7t6s2u: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-mpppvh with version 4.21.37
Unexpected error:
    <*fmt.wrapError | 0x279027f833e0>: 
    failed to create HCP cluster np-version-upgrade-cluster-mpppvh: failed waiting for cluster="np-version-upgrade-cluster-mpppvh" in resourcegroup="rg-np-version-upgrade-mpppvh-6ckrm5njjkhq" to finish creating: GET https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7ef9bc0-8329-437a-9805-6b4b65fd8b47
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: Canceled
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7ef9bc0-8329-437a-9805-6b4b65fd8b47",
      "name": "a7ef9bc0-8329-437a-9805-6b4b65fd8b47",
      "status": "Canceled",
      "startTime": "2026-10-15T04:31:31.170202136Z",
      "endTime": "2026-10-15T04:31:34.719444109Z",
      "error": {
        "code": "Canceled",
        "message": "This operation was superseded by another"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7ef9bc0-8329-437a-9805-6b4b65fd8b47
            correlation ID: bbacf445-8423-4177-9af5-a54728857aa3
            client request ID: 33a84f8d-a321-43e3-a649-cc65e487871e
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-mpppvh with version 4.21.37
Unexpected error:
    <*fmt.wrapError | 0x279027f833e0>: 
    failed to create HCP cluster np-version-upgrade-cluster-mpppvh: failed waiting for cluster="np-version-upgrade-cluster-mpppvh" in resourcegroup="rg-np-version-upgrade-mpppvh-6ckrm5njjkhq" to finish creating: GET https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7ef9bc0-8329-437a-9805-6b4b65fd8b47
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: Canceled
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7ef9bc0-8329-437a-9805-6b4b65fd8b47",
      "name": "a7ef9bc0-8329-437a-9805-6b4b65fd8b47",
      "status": "Canceled",
      "startTime": "2026-10-15T04:31:31.170202136Z",
      "endTime": "2026-10-15T04:31:34.719444109Z",
      "error": {
        "code": "Canceled",
        "message": "This operation was superseded by another"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7ef9bc0-8329-437a-9805-6b4b65fd8b47
            correlation ID: bbacf445-8423-4177-9af5-a54728857aa3
            client request ID: 33a84f8d-a321-43e3-a649-cc65e487871e
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x10cbd284ea60>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T03:26:56.843964Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:26:56.843964Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x10cbd284ea40>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:26:56.843964Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x10cbd155ba40>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x10cbd284ea20>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:26:56.843964Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x10cbd33489f0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x10cbd284ea60>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T03:26:56.843964Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:26:56.843964Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x10cbd284ea40>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:26:56.843964Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x10cbd155ba40>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x10cbd284ea20>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:26:56.843964Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x10cbd33489f0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2a875388c300>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T03:30:50.6897543Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:30:50.6897543Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x2a875388c2e0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:30:50.6897543Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x2a8753a86190>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x2a875388c2a0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:30:50.6897543Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x2a8755208120>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2a875388c300>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T03:30:50.6897543Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:30:50.6897543Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x2a875388c2e0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:30:50.6897543Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x2a8753a86190>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x2a875388c2a0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:30:50.6897543Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x2a8755208120>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T03:48:33Z
  Ended:       2026-10-15T04:04:31Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j0040448-svc", component="backend", name="clusterresources", region="westus3", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
Firing 2:
  State:       Resolved
  Started:     2026-10-15T03:51:31Z
  Ended:       2026-10-15T04:04:31Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j0040448-svc", component="backend", name="csstatedump", region="westus3", severity="warning"
  Description: Backend controller workqueue csstatedump has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5xv7dtgmzwsx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x3a8ed13d18a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5xv7dtgmzwsx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5xv7dtgmzwsx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T03:34:15.9130784Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5xv7dtgmzwsx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5xv7dtgmzwsx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:34:15.9130784Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x3a8ed13d1880>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5xv7dtgmzwsx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:34:15.9130784Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x3a8ed1289130>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x3a8ed13d1860>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5xv7dtgmzwsx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:34:15.9130784Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x3a8ed2d2ad80>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5xv7dtgmzwsx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x3a8ed13d18a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5xv7dtgmzwsx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5xv7dtgmzwsx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T03:34:15.9130784Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5xv7dtgmzwsx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5xv7dtgmzwsx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:34:15.9130784Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x3a8ed13d1880>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5xv7dtgmzwsx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:34:15.9130784Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x3a8ed1289130>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x3a8ed13d1860>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5xv7dtgmzwsx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:34:15.9130784Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x3a8ed2d2ad80>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T04:10:32Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j7932288-mgmt-2", component="kubernetes-infrastructure", container="controller", endpoint="http", environment="ci01", instance="10.128.64.98:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7932288/providers/microsoft.monitor/accounts/hcps-j7932288", namespace="ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w", pod="cloud-network-config-controller-5dbf455969-n498c", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="724a7794-4308-471d-8b59-9caf89cd21b6"
  Description: Pod ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w/cloud-network-config-controller-5dbf455969-n498c (controller) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-15T04:10:32Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j7932288-mgmt-2", component="kubernetes-infrastructure", container="controller", endpoint="http", environment="ci01", instance="10.128.64.98:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7932288/providers/microsoft.monitor/accounts/hcps-j7932288", namespace="ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w", pod="cloud-network-config-controller-5dbf455969-n498c", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="724a7794-4308-471d-8b59-9caf89cd21b6"
  Description: Pod ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w/cloud-network-config-controller-5dbf455969-n498c (controller) is in waiting state (reason: "CrashLoopBackOff").

alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T04:23:32Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7932288-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.98:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7932288/providers/microsoft.monitor/accounts/hcps-j7932288", namespace="ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w", pod="arohcp-monitor-kube-state-metrics-7988dc648c-k89kf", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-15T04:29:35Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7932288-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.98:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7932288/providers/microsoft.monitor/accounts/hcps-j7932288", namespace="ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w", pod="arohcp-monitor-kube-state-metrics-7988dc648c-k89kf", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-q6dds8nqffl9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x39d6a39b4540>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-q6dds8nqffl9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-q6dds8nqffl9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T03:25:39.8200453Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-q6dds8nqffl9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-q6dds8nqffl9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:25:39.8200453Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x39d6a39b4520>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-q6dds8nqffl9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:25:39.8200453Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x39d6a3b305f0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x39d6a39b4500>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-q6dds8nqffl9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:25:39.8200453Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x39d6a5305e00>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-q6dds8nqffl9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x39d6a39b4540>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-q6dds8nqffl9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-q6dds8nqffl9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T03:25:39.8200453Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-q6dds8nqffl9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-q6dds8nqffl9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:25:39.8200453Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x39d6a39b4520>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-q6dds8nqffl9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:25:39.8200453Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x39d6a3b305f0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x39d6a39b4500>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-q6dds8nqffl9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:25:39.8200453Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x39d6a5305e00>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x3c33df13c000>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-15T04:13:54.2670610Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e", ActivityId:"11bf0618-c400-4197-b3c0-0589df8f658a", SubActivityId:"11bf0618-c400-4197-b3c0-0589df8f658a", ActivityType:"GW.Http.CallContext", ParentActivityId:"11bf0618-c400-4197-b3c0-0589df8f658a", ActivityStack:"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a > GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)"}, IsPermanent:false})
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T04:13:54.2670610Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\", ActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", SubActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a > GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\"}, IsPermanent:false})",
        err: <*fmt.wrapError | 0x3c33df13c540>{
            msg: "SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T04:13:54.2670610Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\", ActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", SubActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a > GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\"}, IsPermanent:false})",
            err: <*v2.OneApiError | 0x3c33df6e8000>{
                ErrorMessage: {
                    Code: "LimitsExceeded",
                    Message: "Request is invalid and cannot be executed.",
                    Description: "Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..",
                    Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                    Context: {
                        Timestamp: "2026-10-15T04:13:54.2670610Z",
                        ServiceAlias: "HCP-DEV-US-2",
                        MachineName: "",
                        ProcessName: "",
                        ProcessId: 0,
                        ThreadId: 0,
                        ClientRequestId: "KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e",
                        ActivityId: "11bf0618-c400-4197-b3c0-0589df8f658a",
                        SubActivityId: "11bf0618-c400-4197-b3c0-0589df8f658a",
                        ActivityType: "GW.Http.CallContext",
                        ParentActivityId: "11bf0618-c400-4197-b3c0-0589df8f658a",
                        ActivityStack: "(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a > GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)",
                    },
                    IsPermanent: false,
                },
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x3c33df13c000>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-15T04:13:54.2670610Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e", ActivityId:"11bf0618-c400-4197-b3c0-0589df8f658a", SubActivityId:"11bf0618-c400-4197-b3c0-0589df8f658a", ActivityType:"GW.Http.CallContext", ParentActivityId:"11bf0618-c400-4197-b3c0-0589df8f658a", ActivityStack:"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a > GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)"}, IsPermanent:false})
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T04:13:54.2670610Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\", ActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", SubActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a > GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\"}, IsPermanent:false})",
        err: <*fmt.wrapError | 0x3c33df13c540>{
            msg: "SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T04:13:54.2670610Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\", ActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", SubActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a > GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\"}, IsPermanent:false})",
            err: <*v2.OneApiError | 0x3c33df6e8000>{
                ErrorMessage: {
                    Code: "LimitsExceeded",
                    Message: "Request is invalid and cannot be executed.",
                    Description: "Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..",
                    Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                    Context: {
                        Timestamp: "2026-10-15T04:13:54.2670610Z",
                        ServiceAlias: "HCP-DEV-US-2",
                        MachineName: "",
                        ProcessName: "",
                        ProcessId: 0,
                        ThreadId: 0,
                        ClientRequestId: "KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e",
                        ActivityId: "11bf0618-c400-4197-b3c0-0589df8f658a",
                        SubActivityId: "11bf0618-c400-4197-b3c0-0589df8f658a",
                        ActivityType: "GW.Http.CallContext",
                        ParentActivityId: "11bf0618-c400-4197-b3c0-0589df8f658a",
                        ActivityStack: "(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a > GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)",
                    },
                    IsPermanent: false,
                },
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:108]: failed to get service managed identity
Unexpected error:
    <*exported.ResponseError | 0xf085da10420>: 
    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service
    --------------------------------------------------------------------------------
    RESPONSE 429: 429 Too Many Requests
    ERROR CODE: 429
    --------------------------------------------------------------------------------
    {
      "error": {
        "code": "429",
        "message": "Request is temporarily throttled because tenant XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds."
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service
            correlation ID: 34115d09-ce39-4c55-8529-6ab72e73b469
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:108]: failed to get service managed identity
Unexpected error:
    <*exported.ResponseError | 0xf085da10420>: 
    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service
    --------------------------------------------------------------------------------
    RESPONSE 429: 429 Too Many Requests
    ERROR CODE: 429
    --------------------------------------------------------------------------------
    {
      "error": {
        "code": "429",
        "message": "Request is temporarily throttled because tenant XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds."
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service
            correlation ID: 34115d09-ce39-4c55-8529-6ab72e73b469
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x3527505f6640>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
        err: <*fmt.wrapError | 0x3527505f6620>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x3527505f6640>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
        err: <*fmt.wrapError | 0x3527505f6620>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plddfhjwmv7p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x299d481e23e0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plddfhjwmv7p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plddfhjwmv7p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T03:27:09.5597137Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plddfhjwmv7p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plddfhjwmv7p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:27:09.5597137Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x299d481e23c0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plddfhjwmv7p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:27:09.5597137Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x299d478dc410>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x299d481e23a0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plddfhjwmv7p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:27:09.5597137Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x299d49912a80>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plddfhjwmv7p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x299d481e23e0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plddfhjwmv7p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plddfhjwmv7p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T03:27:09.5597137Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plddfhjwmv7p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plddfhjwmv7p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:27:09.5597137Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x299d481e23c0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plddfhjwmv7p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:27:09.5597137Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x299d478dc410>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x299d481e23a0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plddfhjwmv7p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:27:09.5597137Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x299d49912a80>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x25270548a240>: 
    failed to create NodePool np-deletable: failed waiting for nodepool="np-deletable" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-gzks5p7t2hk9" to finish creating: GET https://rp.j5852928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83ccd50a-e6ff-4e36-bbaf-b33a003914e2
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83ccd50a-e6ff-4e36-bbaf-b33a003914e2",
      "name": "83ccd50a-e6ff-4e36-bbaf-b33a003914e2",
      "status": "Failed",
      "startTime": "2026-10-15T03:39:14.825885335Z",
      "endTime": "2026-10-15T04:08:21.442426792Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine l9m9u8n7x8v1d4q-np-deletable-7glhq-wncf7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-gzks5p7t2hk9--managed/providers/Microsoft.Compute/virtualMachines/l9m9u8n7x8v1d4q-np-deletable-7glhq-wncf7 to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine l9m9u8n7x8v1d4q-np-deletable-7glhq-wncf7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-gzks5p7t2hk9--managed/providers/Microsoft.Compute/virtualMachines/l9m9u8n7x8v1d4q-np-deletable-7glhq-wncf7 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5852928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83ccd50a-e6ff-4e36-bbaf-b33a003914e2
            correlation ID: 23f82d10-9ae2-45ee-9e11-92c0cf8042ec
            client request ID: 53104264-d81f-472b-a94d-ed6c3c435504
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x25270548a240>: 
    failed to create NodePool np-deletable: failed waiting for nodepool="np-deletable" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-gzks5p7t2hk9" to finish creating: GET https://rp.j5852928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83ccd50a-e6ff-4e36-bbaf-b33a003914e2
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83ccd50a-e6ff-4e36-bbaf-b33a003914e2",
      "name": "83ccd50a-e6ff-4e36-bbaf-b33a003914e2",
      "status": "Failed",
      "startTime": "2026-10-15T03:39:14.825885335Z",
      "endTime": "2026-10-15T04:08:21.442426792Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine l9m9u8n7x8v1d4q-np-deletable-7glhq-wncf7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-gzks5p7t2hk9--managed/providers/Microsoft.Compute/virtualMachines/l9m9u8n7x8v1d4q-np-deletable-7glhq-wncf7 to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine l9m9u8n7x8v1d4q-np-deletable-7glhq-wncf7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-gzks5p7t2hk9--managed/providers/Microsoft.Compute/virtualMachines/l9m9u8n7x8v1d4q-np-deletable-7glhq-wncf7 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5852928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83ccd50a-e6ff-4e36-bbaf-b33a003914e2
            correlation ID: 23f82d10-9ae2-45ee-9e11-92c0cf8042ec
            client request ID: 53104264-d81f-472b-a94d-ed6c3c435504

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ftm5cht6nllk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x3bf4ac670420>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ftm5cht6nllk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ftm5cht6nllk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T03:28:42.0427097Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ftm5cht6nllk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ftm5cht6nllk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:28:42.0427097Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x3bf4ac670400>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ftm5cht6nllk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:28:42.0427097Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x3bf4abb46460>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x3bf4ac6703e0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ftm5cht6nllk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:28:42.0427097Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x3bf4ad4f0de0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ftm5cht6nllk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x3bf4ac670420>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ftm5cht6nllk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ftm5cht6nllk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T03:28:42.0427097Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ftm5cht6nllk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ftm5cht6nllk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:28:42.0427097Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x3bf4ac670400>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ftm5cht6nllk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:28:42.0427097Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x3bf4abb46460>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x3bf4ac6703e0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ftm5cht6nllk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:28:42.0427097Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x3bf4ad4f0de0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T04:37:25Z
  Severity:    Sev3
  Labels:      alertname="LeaderElectionLeaseStale", cluster="ci01-j9139968-mgmt-1", component="leader-election", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.96:8080", job="kube-state-metrics", lease="work-manager", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9139968/providers/microsoft.monitor/accounts/services-j9139968", namespace="klusterlet-2tevm554bl2nki54c319ufkm2qrfimd1", pod="arohcp-monitor-kube-state-metrics-7988dc648c-mgv2p", prometheus="prometheus/prometheus", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Leader election lease work-manager in namespace klusterlet-2tevm554bl2nki54c319ufkm2qrfimd1 on cluster ci01-j9139968-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T03:49:50Z
  Ended:       2026-10-15T03:56:50Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j2440448-svc", component="backend", name="actualhostedcluster", region="centralus", severity="warning"
  Description: Backend controller workqueue actualhostedcluster has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.004s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-282j8gjps6m5/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.004s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-282j8gjps6m5/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed

time=2026-10-15T02:14:27.753Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 description="Step mitigations\n Kind: Helm\n"
time=2026-10-15T02:14:28.197Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 values="map[azureCnsCpuLimits:map[enabled:true] jobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157]"
time=2026-10-15T02:14:28.197Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2
time=2026-10-15T02:14:28.543Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 namespace=aro-hcp-mitigations
time=2026-10-15T02:14:28.648Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 dryRun=false
time=2026-10-15T02:14:28.648Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2
time=2026-10-15T02:14:28.690Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2
time=2026-10-15T02:20:27.812Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err="failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1"
time=2026-10-15T02:20:27.812Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err="stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1"

alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T02:26:36Z
  Severity:    Sev3
  Labels:      alertname="MaestroAgentMetricsProxyDown", cluster="ci01-j2839296-mgmt-2", component="monitoring-infrastructure", region="centralus", severity="3"
  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-vbmntn4lpl5c/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-vbmntn4lpl5c/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lq67n7wxjv9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0xa4868a84700>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lq67n7wxjv9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-2teu677bb6kl3jnq3h6qj1u9ribq025b-z3n3z0e0b6i7l6i: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lq67n7wxjv9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-2teu677bb6kl3jnq3h6qj1u9ribq025b-z3n3z0e0b6i7l6i: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        err: <*errors.errorString | 0xa4869d391c0>{
            s: "dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-2teu677bb6kl3jnq3h6qj1u9ribq025b-z3n3z0e0b6i7l6i: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lq67n7wxjv9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0xa4868a84700>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lq67n7wxjv9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-2teu677bb6kl3jnq3h6qj1u9ribq025b-z3n3z0e0b6i7l6i: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lq67n7wxjv9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-2teu677bb6kl3jnq3h6qj1u9ribq025b-z3n3z0e0b6i7l6i: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        err: <*errors.errorString | 0xa4869d391c0>{
            s: "dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-2teu677bb6kl3jnq3h6qj1u9ribq025b-z3n3z0e0b6i7l6i: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-987bhv7wmltw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x239145c4e820>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-987bhv7wmltw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-987bhv7wmltw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T01:45:08.1676917Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-987bhv7wmltw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-987bhv7wmltw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T01:45:08.1676917Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x239145c4e800>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-987bhv7wmltw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T01:45:08.1676917Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x2391450a6140>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x239145c4e7e0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-987bhv7wmltw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T01:45:08.1676917Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x23914738ad80>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-987bhv7wmltw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x239145c4e820>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-987bhv7wmltw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-987bhv7wmltw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T01:45:08.1676917Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-987bhv7wmltw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-987bhv7wmltw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T01:45:08.1676917Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x239145c4e800>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-987bhv7wmltw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T01:45:08.1676917Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x2391450a6140>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x239145c4e7e0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-987bhv7wmltw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T01:45:08.1676917Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x23914738ad80>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred