CIHealth
<
Single Day: Oct 17▾
Pick a day
>
View JSON API
↗ Open Failure patterns for this day

Environment: DEV

Time (UTC)RuntimeJobRegionFailed atResultPRFailed testsDetails
2h 15m 26s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head a5a34b4 · merge a5a34b4
 canadacentral e2e failed ★#7382 (merged) 1
e2e (1)
| take 101, Post "<url>": context deadline exceeded
21:48:55 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x35a206020900>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-17T22:37:32.6448555Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x35a2060208e0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x35a2051a8cd0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x35a2060208c0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x35a206b44300>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x35a206020900>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-17T22:37:32.6448555Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x35a2060208e0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x35a2051a8cd0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x35a2060208c0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x35a206b44300>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
2h 23m 40s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head 7613757 · merge 8849e5d
 centralus alert, e2e failed ⚠#7250 (merged) 2
e2e (1)
RevokeCredentialsAndWait timed out after <minutes> minutes; context deadline exceeded
08:45:27 UTC · failed at e2e · test Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:222]: failed to revoke admin credentials for cluster "admin-cred-lifecycle-mjx8hf"
Unexpected error:
    <*fmt.wrapErrors | 0x1ab46da90510>: 
    failed waiting for hcpCluster="admin-cred-lifecycle-mjx8hf" in resourcegroup="admin-credential-lifecycle-test-k6wptwmmb252" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252, error: context deadline exceeded
    {
        msg: "failed waiting for hcpCluster=\"admin-cred-lifecycle-mjx8hf\" in resourcegroup=\"admin-credential-lifecycle-test-k6wptwmmb252\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0x1ab46cac9c40>{
                s: "timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:222]: failed to revoke admin credentials for cluster "admin-cred-lifecycle-mjx8hf"
Unexpected error:
    <*fmt.wrapErrors | 0x1ab46da90510>: 
    failed waiting for hcpCluster="admin-cred-lifecycle-mjx8hf" in resourcegroup="admin-credential-lifecycle-test-k6wptwmmb252" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252, error: context deadline exceeded
    {
        msg: "failed waiting for hcpCluster=\"admin-cred-lifecycle-mjx8hf\" in resourcegroup=\"admin-credential-lifecycle-test-k6wptwmmb252\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0x1ab46cac9c40>{
                s: "timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred
alert (1)
alert [hcp] KubePodNotReady fired
08:45:27 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubePodNotReady does not fire · suite aro-hcp-tests
Show raw failure
alert fired 32 time(s) (25 unknown, 7 known)
Firing 1:
  State:       Resolved
  Started:     2026-10-17T09:53:56Z
  Ended:       2026-10-17T10:04:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgf9u7qesjgmttof4tbj175kd9c3v6a-j2l3z7n5l1o3m5w", pod="network-node-identity-5479889bd6-wm9n5", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgf9u7qesjgmttof4tbj175kd9c3v6a-j2l3z7n5l1o3m5w/network-node-identity-5479889bd6-wm9n5 has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-17T09:53:56Z
  Ended:       2026-10-17T10:04:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgf9u7qesjgmttof4tbj175kd9c3v6a-j2l3z7n5l1o3m5w", pod="ovnkube-control-plane-6f598f876b-4lwmh", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgf9u7qesjgmttof4tbj175kd9c3v6a-j2l3z7n5l1o3m5w/ovnkube-control-plane-6f598f876b-4lwmh has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-17T09:53:57Z
  Ended:       2026-10-17T10:05:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfa0b8h25qc9a5i20tff4l0fg95ohb-x1q7f4j4x4u2t1g", pod="network-node-identity-5d7f9f545c-9fj88", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfa0b8h25qc9a5i20tff4l0fg95ohb-x1q7f4j4x4u2t1g/network-node-identity-5d7f9f545c-9fj88 has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Resolved
  Started:     2026-10-17T09:56:00Z
  Ended:       2026-10-17T10:05:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfa3i9d0dtsn3mqgm2m0jvbj0gost6-c8m7l6f8p4l9x4f", pod="ovnkube-control-plane-6776f795c7-pnvf4", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfa3i9d0dtsn3mqgm2m0jvbj0gost6-c8m7l6f8p4l9x4f/ovnkube-control-plane-6776f795c7-pnvf4 has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Resolved
  Started:     2026-10-17T09:56:00Z
  Ended:       2026-10-17T10:05:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfa3i9d0dtsn3mqgm2m0jvbj0gost6-c8m7l6f8p4l9x4f", pod="network-node-identity-6484b49989-tpf9j", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfa3i9d0dtsn3mqgm2m0jvbj0gost6-c8m7l6f8p4l9x4f/network-node-identity-6484b49989-tpf9j has been in a non-ready state for longer than 5 minutes.
Firing 6:
  State:       Resolved
  Started:     2026-10-17T09:56:01Z
  Ended:       2026-10-17T10:05:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfacm1jgb7lcrs9ivlei7ps8sq33bk-w6k4h3j3b8i6v3d", pod="network-node-identity-77ddc9bb4c-pbwr7", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfacm1jgb7lcrs9ivlei7ps8sq33bk-w6k4h3j3b8i6v3d/network-node-identity-77ddc9bb4c-pbwr7 has been in a non-ready state for longer than 5 minutes.
Firing 7:
  State:       Resolved
  Started:     2026-10-17T09:56:01Z
  Ended:       2026-10-17T10:04:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfa5d10e98e3mrvr8mdtggsfec8ur7-z9i7c7u7w1a3k9w", pod="etcd-1", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfa5d10e98e3mrvr8mdtggsfec8ur7-z9i7c7u7w1a3k9w/etcd-1 has been in a non-ready state for longer than 5 minutes.
Firing 8:
  State:       Resolved
  Started:     2026-10-17T09:56:01Z
  Ended:       2026-10-17T10:05:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfacm1jgb7lcrs9ivlei7ps8sq33bk-w6k4h3j3b8i6v3d", pod="ovnkube-control-plane-6bd64ff8dc-l5xp2", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfacm1jgb7lcrs9ivlei7ps8sq33bk-w6k4h3j3b8i6v3d/ovnkube-control-plane-6bd64ff8dc-l5xp2 has been in a non-ready state for longer than 5 minutes.
Firing 9:
  State:       Resolved
  Started:     2026-10-17T09:58:01Z
  Ended:       2026-10-17T10:05:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfacmm0dlphe6vtc50mkmtd3jb2p8q-f2o6b4o3v3g6y5w", pod="kube-apiserver-9875495b7-tqsz4", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfacmm0dlphe6vtc50mkmtd3jb2p8q-f2o6b4o3v3g6y5w/kube-apiserver-9875495b7-tqsz4 has been in a non-ready state for longer than 5 minutes.
Firing 10:
  State:       Resolved
  Started:     2026-10-17T09:58:01Z
  Ended:       2026-10-17T10:05:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfacmm0dlphe6vtc50mkmtd3jb2p8q-f2o6b4o3v3g6y5w", pod="kube-apiserver-9875495b7-222sx", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfacmm0dlphe6vtc50mkmtd3jb2p8q-f2o6b4o3v3g6y5w/kube-apiserver-9875495b7-222sx has been in a non-ready state for longer than 5 minutes.
Firing 11:
  State:       Resolved
  Started:     2026-10-17T09:58:01Z
  Ended:       2026-10-17T10:04:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfadvi8jv413ort809d0n4kq5kno22-s2q4c0i7f2c0r2x", pod="network-node-identity-5d8fccd895-ptmvr", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfadvi8jv413ort809d0n4kq5kno22-s2q4c0i7f2c0r2x/network-node-identity-5d8fccd895-ptmvr has been in a non-ready state for longer than 5 minutes.
Firing 12:
  State:       Resolved
  Started:     2026-10-17T09:58:01Z
  Ended:       2026-10-17T10:05:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfacmauure3rehr0ll3kf67lkonqn4-l7q9g3b3j1r5c8t", pod="network-node-identity-86d7bc77f4-zzxmb", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfacmauure3rehr0ll3kf67lkonqn4-l7q9g3b3j1r5c8t/network-node-identity-86d7bc77f4-zzxmb has been in a non-ready state for longer than 5 minutes.
Firing 13:
  State:       Resolved
  Started:     2026-10-17T09:58:01Z
  Ended:       2026-10-17T10:05:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfacmm0dlphe6vtc50mkmtd3jb2p8q-f2o6b4o3v3g6y5w", pod="kube-apiserver-9875495b7-scjqk", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfacmm0dlphe6vtc50mkmtd3jb2p8q-f2o6b4o3v3g6y5w/kube-apiserver-9875495b7-scjqk has been in a non-ready state for longer than 5 minutes.
Firing 14:
  State:       Resolved
  Started:     2026-10-17T09:58:58Z
  Ended:       2026-10-17T10:05:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfacoq552p37e3lj875ff1g01eve2p-j5s8a2u8s3o2v6k", pod="kube-apiserver-5844c8889f-cbz2x", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfacoq552p37e3lj875ff1g01eve2p-j5s8a2u8s3o2v6k/kube-apiserver-5844c8889f-cbz2x has been in a non-ready state for longer than 5 minutes.
Firing 15:
  State:       Resolved
  Started:     2026-10-17T09:58:58Z
  Ended:       2026-10-17T10:05:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfacoq552p37e3lj875ff1g01eve2p-j5s8a2u8s3o2v6k", pod="kube-apiserver-5844c8889f-d6fvz", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfacoq552p37e3lj875ff1g01eve2p-j5s8a2u8s3o2v6k/kube-apiserver-5844c8889f-d6fvz has been in a non-ready state for longer than 5 minutes.
Firing 16:
  State:       Resolved
  Started:     2026-10-17T09:58:58Z
  Ended:       2026-10-17T10:05:55Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfa3iukddj4rbptbdr63ko16fjd5a0-private-kas", pod="kube-apiserver-8544f679b7-zl64m", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfa3iukddj4rbptbdr63ko16fjd5a0-private-kas/kube-apiserver-8544f679b7-zl64m has been in a non-ready state for longer than 5 minutes.
Firing 17:
  State:       Resolved
  Started:     2026-10-17T09:58:58Z
  Ended:       2026-10-17T10:05:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfa70s9eurd03qajc81h11klkgrmcp-q6y7b0q2i4d5g1m", pod="network-node-identity-6cd587db9f-t4z8h", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfa70s9eurd03qajc81h11klkgrmcp-q6y7b0q2i4d5g1m/network-node-identity-6cd587db9f-t4z8h has been in a non-ready state for longer than 5 minutes.
Firing 18:
  State:       Resolved
  Started:     2026-10-17T09:58:58Z
  Ended:       2026-10-17T10:05:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfa3iukddj4rbptbdr63ko16fjd5a0-private-kas", pod="kube-apiserver-8544f679b7-5fcvk", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfa3iukddj4rbptbdr63ko16fjd5a0-private-kas/kube-apiserver-8544f679b7-5fcvk has been in a non-ready state for longer than 5 minutes.
Firing 19:
  State:       Resolved
  Started:     2026-10-17T09:58:58Z
  Ended:       2026-10-17T10:04:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfadpgoo9rvetrl85b24o6ppv4j0fo-o3i3s1q6c0r8x5c", pod="network-node-identity-5d97d9b5cb-4k88v", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfadpgoo9rvetrl85b24o6ppv4j0fo-o3i3s1q6c0r8x5c/network-node-identity-5d97d9b5cb-4k88v has been in a non-ready state for longer than 5 minutes.
Firing 20:
  State:       Resolved
  Started:     2026-10-17T09:58:58Z
  Ended:       2026-10-17T10:04:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfacn5q7o0r4p4o2djer1ifh59adg8-ea-listq25ppj", pod="network-node-identity-74f67fb4bf-ngwc7", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfacn5q7o0r4p4o2djer1ifh59adg8-ea-listq25ppj/network-node-identity-74f67fb4bf-ngwc7 has been in a non-ready state for longer than 5 minutes.
Firing 21:
  State:       Resolved
  Started:     2026-10-17T09:58:58Z
  Ended:       2026-10-17T10:05:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfacoq552p37e3lj875ff1g01eve2p-j5s8a2u8s3o2v6k", pod="kube-apiserver-5844c8889f-b42qs", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfacoq552p37e3lj875ff1g01eve2p-j5s8a2u8s3o2v6k/kube-apiserver-5844c8889f-b42qs has been in a non-ready state for longer than 5 minutes.
Firing 22:
  State:       Resolved
  Started:     2026-10-17T09:58:58Z
  Ended:       2026-10-17T10:04:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfapf0dhnoj0q238ct23l9vk8nc5at-k7b2q6t5f3g5p6n", pod="network-node-identity-fb54b4d4d-v8wgh", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfapf0dhnoj0q238ct23l9vk8nc5at-k7b2q6t5f3g5p6n/network-node-identity-fb54b4d4d-v8wgh has been in a non-ready state for longer than 5 minutes.
Firing 23:
  State:       Resolved
  Started:     2026-10-17T09:58:58Z
  Ended:       2026-10-17T10:04:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfadpgoo9rvetrl85b24o6ppv4j0fo-o3i3s1q6c0r8x5c", pod="ovnkube-control-plane-6895bcbdc6-c9kvm", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfadpgoo9rvetrl85b24o6ppv4j0fo-o3i3s1q6c0r8x5c/ovnkube-control-plane-6895bcbdc6-c9kvm has been in a non-ready state for longer than 5 minutes.
Firing 24:
  State:       Resolved
  Started:     2026-10-17T09:58:59Z
  Ended:       2026-10-17T10:05:55Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfa3iukddj4rbptbdr63ko16fjd5a0-private-kas", pod="kube-apiserver-8544f679b7-q8rkz", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfa3iukddj4rbptbdr63ko16fjd5a0-private-kas/kube-apiserver-8544f679b7-q8rkz has been in a non-ready state for longer than 5 minutes.
Firing 25:
  State:       Resolved
  Started:     2026-10-17T09:59:55Z
  Ended:       2026-10-17T10:05:56Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0319872-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tgfa1d4f46ij8t9ghuvhfoa11l51bep-basic-cluster", pod="network-node-identity-5774b9669-l5ggc", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tgfa1d4f46ij8t9ghuvhfoa11l51bep-basic-cluster/network-node-identity-5774b9669-l5ggc has been in a non-ready state for longer than 5 minutes.