{"meta":{"start_date":"2026-10-14","end_date":"2026-10-20","start_at":"2026-10-14T00:00:00Z","end_at":"2026-10-21T00:00:00Z","timezone":"UTC","generated_at":"2026-10-15T08:11:46Z","environments":["dev","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":140,"failed_runs":113,"raw_occurrences":207,"matched_occurrences":200,"runs_affected":106},"rows":[{"environment":"dev","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":34,"runs_affected":34,"failed_runs":34,"run_impact_percent":24.285714285714285,"anchor_occurrences":34,"after_last_push_count":5,"trend_counts":[2,26,8,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":36,"prior_last_seen_at":"2026-10-15T05:42:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":34}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d47db71ece0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T06:26:16.8623213Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | join kind=leftsemi hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | join kind=leftsemi hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-15T06:26:16.8623213Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1d47db71eca0\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-15T06:26:16.8623213Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x1d47d9c628c0\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x1d47db71ec80\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-15T06:26:16.8623213Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x1d47d9ddf1d0\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d47db71ece0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T06:26:16.8623213Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | join kind=leftsemi hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | join kind=leftsemi hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-15T06:26:16.8623213Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1d47db71eca0\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-15T06:26:16.8623213Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x1d47d9c628c0\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x1d47db71ec80\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5hsqz97rck5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-15T06:26:16.8623213Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x1d47d9ddf1d0\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x10cbd284ea60\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:26:56.843964Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | join kind=leftsemi hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | join kind=leftsemi hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-15T03:26:56.843964Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x10cbd284ea40\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-15T03:26:56.843964Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x10cbd155ba40\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x10cbd284ea20\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-15T03:26:56.843964Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x10cbd33489f0\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x10cbd284ea60\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:26:56.843964Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | join kind=leftsemi hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | join kind=leftsemi hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-15T03:26:56.843964Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x10cbd284ea40\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-15T03:26:56.843964Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x10cbd155ba40\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x10cbd284ea20\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wg8m45z4ln98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-15T03:26:56.843964Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x10cbd33489f0\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a875388c300\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:30:50.6897543Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | join kind=leftsemi hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | join kind=leftsemi hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-15T03:30:50.6897543Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2a875388c2e0\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-15T03:30:50.6897543Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x2a8753a86190\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x2a875388c2a0\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-15T03:30:50.6897543Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x2a8755208120\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a875388c300\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T03:30:50.6897543Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | join kind=leftsemi hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | join kind=leftsemi hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-15T03:30:50.6897543Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2a875388c2e0\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-15T03:30:50.6897543Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x2a8753a86190\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x2a875388c2a0\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tvswt787smnh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-15T03:30:50.6897543Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x2a8755208120\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7487/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110607153617178624","occurred_at":"2026-10-15T05:42:13Z","signature_id":"f41e6789e6b26c9adf70978b7436d49fb8bbf2b1fc6cc62fa3dffca6d08aa5fc","pr_number":7487,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7546/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110562220516577280","occurred_at":"2026-10-15T02:43:37Z","signature_id":"45c4f42cbc3394e8d69b5ccff8a904b2f9c2403ade9ecdfd1010b4b987308c68","pr_number":7546,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7498/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558456200040448","occurred_at":"2026-10-15T02:28:42Z","signature_id":"1949375ff2b581247bba104a1ddd7d7e7b539444467f08076c41b3a836b79e5c","pr_number":7498,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558444497932288","occurred_at":"2026-10-15T02:28:40Z","signature_id":"dfca8e66f25d1dc47b86b0bdd51891ee8106201ba523c149c3a5bcaa770a0fe5","pr_number":7492,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7499/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558435291435008","occurred_at":"2026-10-15T02:28:37Z","signature_id":"4ab7144c5c500292b3e3173f19ed52ce36626055729a5b45c564a3a540c45a66","pr_number":7499,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558373865852928","occurred_at":"2026-10-15T02:28:23Z","signature_id":"4a98ab6d29d4ac6ec2b8ed5c01e37d793c68fc2131bcd10feecd606b87143b86","pr_number":7524,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558354509139968","occurred_at":"2026-10-15T02:28:18Z","signature_id":"ceca02b107a69f0963dfed1a5c6ae5b5b588366f559e0f44fcbdb03e555d8ea5","pr_number":7509,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110534454291206144","occurred_at":"2026-10-15T00:53:20Z","signature_id":"e24fe37523dd8477f2e9248cefea3d57f206a693446feef12e700343453024ae","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7532/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110513999702396928","occurred_at":"2026-10-14T23:32:03Z","signature_id":"e61871927b40d79ab621de3d7b04db3309cb36b6da4099f014a1287bcddd206a","pr_number":7532,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110500986920374272","occurred_at":"2026-10-14T22:40:21Z","signature_id":"8b6ff30d5729c06336599627d1753d772466979c7d321edead0ce4abc023e2f2","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110500816379973632","occurred_at":"2026-10-14T22:39:40Z","signature_id":"df64f40d70cc038ea7e186e9fe3ec3ba843b2428158b72034bbd34caf4fb5367","pr_number":7342,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110483415672295424","occurred_at":"2026-10-14T21:30:23Z","signature_id":"ea54a393ba27dfd62e41cfe9e83dadd2af63caf674e37f0a01a3ab250f594c72","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110456066088112128","occurred_at":"2026-10-14T19:41:51Z","signature_id":"5724b1afa43f955853aa69aef7840da478df768f7adac5bef575c3065271b7e3","pr_number":7502,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7498/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452354565279744","occurred_at":"2026-10-14T19:27:06Z","signature_id":"7979b20855eabda20007f6470ef29f513c1d6135e9e8a5039320fb7f4ffa22f4","pr_number":7498,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452322852147200","occurred_at":"2026-10-14T19:26:58Z","signature_id":"61be387220ef87ff768c357b2113308f70f1e46f3b2c65bbda3df6db4746431d","pr_number":7497,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452284272939008","occurred_at":"2026-10-14T19:26:49Z","signature_id":"8a6c53fba1289eafddee3dc2209da4cc4da34923949b74e88f7af84c0c16efbe","pr_number":7500,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110444073914470400","occurred_at":"2026-10-14T18:54:11Z","signature_id":"26cc3f18c688e60a4525b3a85a9535f2ebbb9f5b82f993744500f76041d5e350","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7527/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110408382648160256","occurred_at":"2026-10-14T16:32:22Z","signature_id":"3407f3c6628610125e781d12dcb7b736191590fb6f84e2cb2605a7040e22e20e","pr_number":7527,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110407002151718912","occurred_at":"2026-10-14T16:26:53Z","signature_id":"03beb3e937976fe9c9cc83f1032b6fe1c93c15757c83dbb637e830643a12476e","pr_number":7524,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110389003348873216","occurred_at":"2026-10-14T15:15:22Z","signature_id":"600f850e5b0064c6e16a842f50c39fbcf04bb8615687624b565925b81e9272cd","pr_number":7500,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110371287573467136","occurred_at":"2026-10-14T14:04:58Z","signature_id":"446971e32c8d7eaa419e50808053148289030b264b2065526ee8ad44598d6bc2","pr_number":7524,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110361307805913088","occurred_at":"2026-10-14T13:25:18Z","signature_id":"f21ed846cfee8ba8329873d162dbbca110a0827914a71233b54bd9b0e887e5d1","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"00d6afff568ec9fe8320e006deebf9a2ec333eeafd0113a7207c3401059e8288","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110334902741241856","occurred_at":"2026-10-14T11:40:23Z","signature_id":"33bbace3f82b158f3b3934cc2b9a2c887834084f3072bc89bf02086c9319be95","pr_number":7500,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110334900086247424","occurred_at":"2026-10-14T11:40:22Z","signature_id":"1fd2861a2ecdecb50432c0aaea802d173b55f8803b47a00d6ba6388a6f98a3c1","pr_number":7492,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110334374363795456","occurred_at":"2026-10-14T11:38:17Z","signature_id":"3e29d39bd8590975516a2c5d3e94fd98154d3dfc49c7ae3cbae6f51d43e795ef","pr_number":7502,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110305876723109888","occurred_at":"2026-10-14T09:44:59Z","signature_id":"84b1932c2aad8c16a68cba4715462d64dc0cf19c6b12f77659cab41f2c157772","pr_number":7497,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110269833781186560","occurred_at":"2026-10-14T07:21:49Z","signature_id":"9cffe9e337d6e68d2d1815dd3533f556856d9bf85b4da15334b89c4b7a8023d2","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7518/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110268638186442752","occurred_at":"2026-10-14T07:16:28Z","signature_id":"31fcba85fdd69c41767174d3dfec81cf15e72884a0829fbd9bb13aa3cf481d6d","pr_number":7518,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110267140211740672","occurred_at":"2026-10-14T07:11:07Z","signature_id":"39b08081edae55a01b00ee6d278d4fe5b4565d42932b3eb0793613aa32d74b95","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110267078714855424","occurred_at":"2026-10-14T07:10:50Z","signature_id":"84ff6b9fa357efbe37c1955bfcfcc7167434d092376b6767fb221f43e1ee5f81","pr_number":7519,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110235250528358400","occurred_at":"2026-10-14T05:04:24Z","signature_id":"388fa5f884dc9873a3e0ac5e7934c037c1b2aca0708794c9638a9509d5d20e5f","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110213970672488448","occurred_at":"2026-10-14T03:39:51Z","signature_id":"c3f223ea41e95da8d49b0cc04384dc9d6bba824c5592c8a1fc0a741ad927ca6d","pr_number":7504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110168566228586496","occurred_at":"2026-10-14T00:39:25Z","signature_id":"d34b77d805ee032e385bd7e9ec93a0f17a9c99b3eb797f5536963db648176fbe","pr_number":7248,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":15,"runs_affected":15,"failed_runs":15,"run_impact_percent":10.714285714285714,"anchor_occurrences":15,"after_last_push_count":67,"trend_counts":[9,12,3,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":219,"prior_last_seen_at":"2026-10-15T04:29:39Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":15}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-15T05:33:44Z\n  Ended:       2026-10-15T05:53:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8670976-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-15T03:48:33Z\n  Ended:       2026-10-15T04:04:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0040448-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-15T03:51:31Z\n  Ended:       2026-10-15T04:04:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0040448-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-15T03:49:50Z\n  Ended:       2026-10-15T03:56:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2440448-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"centralus\", severity=\"warning\"\n  Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110588894188670976","occurred_at":"2026-10-15T04:29:39Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7498/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558456200040448","occurred_at":"2026-10-15T02:28:42Z","signature_id":"b398e3b4b60ce0bfe92ea3c19658030c4ec5dee4a44f8a5f5c5664c590af5113","pr_number":7498,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558351342440448","occurred_at":"2026-10-15T02:28:17Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110505977575903232","occurred_at":"2026-10-14T23:00:10Z","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7526/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110504372508037120","occurred_at":"2026-10-14T22:53:42Z","signature_id":"1055a81696a8c72ee71de5548a9db39421bfe798741063f2b4483fcbb49ce2e8","pr_number":7526,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110500986920374272","occurred_at":"2026-10-14T22:40:21Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7528/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110483351910486016","occurred_at":"2026-10-14T21:30:15Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7528,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110483335657558016","occurred_at":"2026-10-14T21:30:12Z","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","pr_number":7524,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6866/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110463708953055232","occurred_at":"2026-10-14T20:12:13Z","signature_id":"ec53f16855846d8bcc63c32e61dda8320f5f786dcafa5b90527c9de5094abd44","pr_number":6866,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110456066088112128","occurred_at":"2026-10-14T19:41:51Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7502,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110412598238253056","occurred_at":"2026-10-14T16:49:06Z","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","pr_number":7529,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7428/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110376503379234816","occurred_at":"2026-10-14T14:25:41Z","signature_id":"b398e3b4b60ce0bfe92ea3c19658030c4ec5dee4a44f8a5f5c5664c590af5113","pr_number":7428,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7522/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110350863582105600","occurred_at":"2026-10-14T12:43:47Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7522,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110303886584582144","occurred_at":"2026-10-14T09:37:08Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7383,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110265029910597632","occurred_at":"2026-10-14T07:02:43Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7516,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":13,"runs_affected":13,"failed_runs":13,"run_impact_percent":9.285714285714286,"anchor_occurrences":13,"after_last_push_count":20,"trend_counts":[6,11,2,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":81,"prior_last_seen_at":"2026-10-15T05:43:27Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":13}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T07:28:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7707648-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.196:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7707648/providers/microsoft.monitor/accounts/hcps-j7707648\", namespace=\"ocm-arohcpci01-2tf236h3grf5nh5cka2klv381nroha64-des-encrypt\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-wd85h\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tf236h3grf5nh5cka2klv381nroha64-des-encrypt/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T07:28:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7707648-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.196:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7707648/providers/microsoft.monitor/accounts/hcps-j7707648\", namespace=\"ocm-arohcpci01-2tf236h3grf5nh5cka2klv381nroha64-des-encrypt\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-wd85h\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tf236h3grf5nh5cka2klv381nroha64-des-encrypt/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T07:28:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7707648-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.196:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7707648/providers/microsoft.monitor/accounts/hcps-j7707648\", namespace=\"ocm-arohcpci01-2tf236h3grf5nh5cka2klv381nroha64-des-encrypt\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-wd85h\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tf236h3grf5nh5cka2klv381nroha64-des-encrypt/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T07:28:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7707648-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.196:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7707648/providers/microsoft.monitor/accounts/hcps-j7707648\", namespace=\"ocm-arohcpci01-2tf236h3grf5nh5cka2klv381nroha64-des-encrypt\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-wd85h\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tf236h3grf5nh5cka2klv381nroha64-des-encrypt/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T04:23:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7932288-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.98:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7932288/providers/microsoft.monitor/accounts/hcps-j7932288\", namespace=\"ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-k89kf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T04:29:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7932288-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.98:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7932288/providers/microsoft.monitor/accounts/hcps-j7932288\", namespace=\"ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-k89kf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-14T23:39:28Z\n  Ended:       2026-10-14T23:53:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0214528-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0214528/providers/microsoft.monitor/accounts/hcps-j0214528\", namespace=\"ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-r2hbm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-14T23:39:28Z\n  Ended:       2026-10-14T23:53:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0214528-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0214528/providers/microsoft.monitor/accounts/hcps-j0214528\", namespace=\"ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-r2hbm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u/capi-provider has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110607466327707648","occurred_at":"2026-10-15T05:43:27Z","signature_id":"63ad5d8eb64723b20ad5d3cef316adc16a92e54d0efbf2073890cf7cec3f0852","pr_number":7509,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558444497932288","occurred_at":"2026-10-15T02:28:40Z","signature_id":"6ecefab153f490fd537d46ce93ebe675f7ec3355f24754aee618349d423b68c2","pr_number":7492,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110494690280214528","occurred_at":"2026-10-14T22:15:16Z","signature_id":"136d96d86a4c895fcd3c1d444abd7575070d82b11f64ab40d965df68d6fdaaf5","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110445968326397952","occurred_at":"2026-10-14T19:01:26Z","signature_id":"f40a39c9ca10509006eb38c52071e0f722fbe1b343f8d8790c565126c0289f61","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110437162552922112","occurred_at":"2026-10-14T18:26:44Z","signature_id":"cab2c47a06c7ff0ab3e837d0af7e5dadc7a44e0f4414274b4500f2b8eaa58ae9","pr_number":7524,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7301/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110412681776205824","occurred_at":"2026-10-14T16:49:27Z","signature_id":"a8e82832880b9ffa15cd92fbe758dc94bf292f90b84ee2a2b3b8eb7fbe84ceab","pr_number":7301,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7428/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110376503379234816","occurred_at":"2026-10-14T14:25:41Z","signature_id":"ae392ace2ed0c8022d2929768cf5177dca0ebce5bf93ae7f0d81e46d65074045","pr_number":7428,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"55034fbd6d3c5e94b22e5c438df96ac5629b046d02f4f8f6a566e208fb622951","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110334900086247424","occurred_at":"2026-10-14T11:40:22Z","signature_id":"55c67bf5d39db25b9b8b07942180b2686cfe0e47355a56931e067d63af27c315","pr_number":7492,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110291728471691264","occurred_at":"2026-10-14T08:48:48Z","signature_id":"9ba21becf373fa33475cb896a527b1cbf47fc159b36b23de4d6dc7754df75409","pr_number":7500,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110269833781186560","occurred_at":"2026-10-14T07:21:49Z","signature_id":"84eef7d5861896e04804ef44919b3f5111909b2b4c789a45fce7492b63e3a407","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110265029910597632","occurred_at":"2026-10-14T07:02:43Z","signature_id":"8c2aff0d3632c81340ce3a2c0cb498d1dab4f667e7346d1f4af0133ec58e4f5f","pr_number":7516,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110221944149250048","occurred_at":"2026-10-14T04:11:31Z","signature_id":"7756294fda68fe98e6d3f7d273b6b789955b562628b9575f65001f0516fb17ea","pr_number":6712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":10,"runs_affected":10,"failed_runs":10,"run_impact_percent":7.142857142857143,"anchor_occurrences":10,"after_last_push_count":20,"trend_counts":[13,8,2,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":52,"prior_last_seen_at":"2026-10-15T04:29:39Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":10}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T06:10:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j8670976-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.103:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8670976/providers/microsoft.monitor/accounts/services-j8670976\", namespace=\"klusterlet-2tf18dgs468rqeforqnek754iblq2hqg\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-4r5xr\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2tf18dgs468rqeforqnek754iblq2hqg on cluster ci01-j8670976-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T04:37:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j9139968-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.96:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9139968/providers/microsoft.monitor/accounts/services-j9139968\", namespace=\"klusterlet-2tevm554bl2nki54c319ufkm2qrfimd1\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgv2p\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2tevm554bl2nki54c319ufkm2qrfimd1 on cluster ci01-j9139968-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T00:52:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j2275200-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.169:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2275200/providers/microsoft.monitor/accounts/services-j2275200\", namespace=\"klusterlet-2tesd0o3f60ell7kn84k7na2ifcblhbl\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-wclqz\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2tesd0o3f60ell7kn84k7na2ifcblhbl on cluster ci01-j2275200-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110588894188670976","occurred_at":"2026-10-15T04:29:39Z","signature_id":"1d84b1a7cfc29b2eaa012fb4fc792bc3e3ed7989684d13143ab9e7705252ea88","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558354509139968","occurred_at":"2026-10-15T02:28:18Z","signature_id":"6368b0c6fa0c1de7f3094acf13f7071ec55cea65182b85f7bbf296ca29822fb2","pr_number":7509,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7540/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110508385492275200","occurred_at":"2026-10-14T23:09:45Z","signature_id":"3c96117c5ddc1261bfec64ae3985a8f8906d46b2a771385f7492378201af02c6","pr_number":7540,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7538/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110474243606056960","occurred_at":"2026-10-14T20:54:04Z","signature_id":"d0ae0ee40a93e9522e8aa3aad5e2e8d13a87da569f13779e1b12906aa8c9b53c","pr_number":7538,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452322852147200","occurred_at":"2026-10-14T19:26:58Z","signature_id":"76b6f718f1f07d60590d0948a4fdfeda4a1da54d0c8b8a64b4f169c6a0d37c98","pr_number":7497,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110437162552922112","occurred_at":"2026-10-14T18:26:44Z","signature_id":"423d326e2b1972473d10fe07604ba7cb4534248683d2ec096f5ae5959abcce41","pr_number":7524,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7527/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110408382648160256","occurred_at":"2026-10-14T16:32:22Z","signature_id":"fcf669af0e8c7da39eaec1b22f419c476a115f903e8455b3aff82defb226e02c","pr_number":7527,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7525/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110338041116626944","occurred_at":"2026-10-14T11:52:51Z","signature_id":"a5775537c5d4a02a7df57596f722ad13cd1cea1e62fcb6e5097655900ef1ec2b","pr_number":7525,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7499/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110304557253791744","occurred_at":"2026-10-14T09:39:48Z","signature_id":"f560bfc31c39b650b322b862f5a16de94b12597c676328f7859afb2592dc53de","pr_number":7499,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110272422715330560","occurred_at":"2026-10-14T07:30:12Z","signature_id":"9f5a880ad682e61c10851eb2f1014abff11271943a08f54b06f8a578999bf9d7","pr_number":7441,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-282j8gjps6m5/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":9,"runs_affected":9,"failed_runs":9,"run_impact_percent":6.428571428571429,"anchor_occurrences":9,"after_last_push_count":19,"trend_counts":[6,7,2,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":47,"prior_last_seen_at":"2026-10-15T02:14:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":9}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-282j8gjps6m5/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-282j8gjps6m5/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-vbmntn4lpl5c/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-vbmntn4lpl5c/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-44v5fj7gn2vn/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-44v5fj7gn2vn/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7544/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110554897882550272","occurred_at":"2026-10-15T02:14:34Z","signature_id":"c3f5ffc66171aee04e4ea896b6fd7ad1f5c40c428f2cc854d4cacf8b7ba6cad1","pr_number":7544,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110543177340424192","occurred_at":"2026-10-15T01:28:00Z","signature_id":"8c8792c48ba5bf46de2e19623688298335ee9162e2cbfd1034ac375c4ecb3c4c","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7544/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110520946510008320","occurred_at":"2026-10-14T23:59:39Z","signature_id":"dbdad5bcd6883b547539aa2a7128d9d6dac9b47d512abf71a69bd30baf31f61b","pr_number":7544,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7540/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110508385492275200","occurred_at":"2026-10-14T23:09:45Z","signature_id":"92f3748303a061cc6184e933650c30db9f661fd5374936ed7875565522f2b06d","pr_number":7540,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110490547654234112","occurred_at":"2026-10-14T21:58:50Z","signature_id":"5051886dbfc3c36339c7c0eb02b10fb512049eb5a0ac03511948fd813a8b24fa","pr_number":7492,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452284272939008","occurred_at":"2026-10-14T19:26:49Z","signature_id":"36fb74339a311618119525c91dbbd8ddfaca88e17b00be4e06ba05c2588ce610","pr_number":7500,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110407002151718912","occurred_at":"2026-10-14T16:26:53Z","signature_id":"add2ded45d4e7d255ace8de8f5215174d5761f231ca7fb8a87f2550584249b6b","pr_number":7524,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110213970672488448","occurred_at":"2026-10-14T03:39:51Z","signature_id":"fc73da3f64df28e28a5f3d03ec006882753ad077ad7fd997dd91714a48b160a9","pr_number":7504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110191032388816896","occurred_at":"2026-10-14T02:08:42Z","signature_id":"aba6bac3ecf6fccefb4d5d45429681cbe92d317984bf96d8fe7f780c1e7bca9a","pr_number":6968,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":9,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.4285714285714286,"anchor_occurrences":9,"after_last_push_count":1,"trend_counts":[0,9,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-12"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-14T18:22:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE starts a control plane z-stream upgrade with an SRE pin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:114]: failed to create cluster \"sre-version-pin-vc9jq7\" at 4.21.36\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2da62fe74520\u003e: \n    failed to create HCP cluster sre-version-pin-vc9jq7: failed waiting for cluster=\"sre-version-pin-vc9jq7\" in resourcegroup=\"admin-api-version-pin-ncqjrnsn47gn\" to finish creating: GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3a2cf661-5481-4ce5-9c2c-34f8ee3716ac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3a2cf661-5481-4ce5-9c2c-34f8ee3716ac\",\n      \"name\": \"3a2cf661-5481-4ce5-9c2c-34f8ee3716ac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T19:11:51.389728795Z\",\n      \"endTime\": \"2026-10-14T19:41:00.041259206Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3a2cf661-5481-4ce5-9c2c-34f8ee3716ac\n            correlation ID: 3bd031da-51c1-4750-a9ad-a1b50d360688\n            client request ID: c18e0ef2-a2d7-4b12-a856-0335c427f46a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:114]: failed to create cluster \"sre-version-pin-vc9jq7\" at 4.21.36\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2da62fe74520\u003e: \n    failed to create HCP cluster sre-version-pin-vc9jq7: failed waiting for cluster=\"sre-version-pin-vc9jq7\" in resourcegroup=\"admin-api-version-pin-ncqjrnsn47gn\" to finish creating: GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3a2cf661-5481-4ce5-9c2c-34f8ee3716ac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3a2cf661-5481-4ce5-9c2c-34f8ee3716ac\",\n      \"name\": \"3a2cf661-5481-4ce5-9c2c-34f8ee3716ac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T19:11:51.389728795Z\",\n      \"endTime\": \"2026-10-14T19:41:00.041259206Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3a2cf661-5481-4ce5-9c2c-34f8ee3716ac\n            correlation ID: 3bd031da-51c1-4750-a9ad-a1b50d360688\n            client request ID: c18e0ef2-a2d7-4b12-a856-0335c427f46a\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:152]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3dad90af33a0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-sbj86zcf4vq4\" to finish creating: GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/18cf34f6-b63a-4010-8930-865b2da0c38d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/18cf34f6-b63a-4010-8930-865b2da0c38d\",\n      \"name\": \"18cf34f6-b63a-4010-8930-865b2da0c38d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T19:09:44.204708518Z\",\n      \"endTime\": \"2026-10-14T19:38:50.032848393Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/18cf34f6-b63a-4010-8930-865b2da0c38d\n            correlation ID: 8581e472-de79-4013-9c83-cf00277e2806\n            client request ID: 3009aa8f-ca8e-40df-af26-d6a54c07c012\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:152]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3dad90af33a0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-sbj86zcf4vq4\" to finish creating: GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/18cf34f6-b63a-4010-8930-865b2da0c38d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/18cf34f6-b63a-4010-8930-865b2da0c38d\",\n      \"name\": \"18cf34f6-b63a-4010-8930-865b2da0c38d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T19:09:44.204708518Z\",\n      \"endTime\": \"2026-10-14T19:38:50.032848393Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/18cf34f6-b63a-4010-8930-865b2da0c38d\n            correlation ID: 8581e472-de79-4013-9c83-cf00277e2806\n            client request ID: 3009aa8f-ca8e-40df-af26-d6a54c07c012\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-mxm2nz\" with version 4.21.36 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e81b57e6100\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-mxm2nz: failed waiting for cluster=\"cluster-zstream-4-21-mxm2nz\" in resourcegroup=\"rg-zstream-upgrade-4-21-9f22s7ghtq4x\" to finish creating: GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e22f032f-fd1f-4f7a-a0f0-79ca0d681ff5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e22f032f-fd1f-4f7a-a0f0-79ca0d681ff5\",\n      \"name\": \"e22f032f-fd1f-4f7a-a0f0-79ca0d681ff5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T19:09:23.643995805Z\",\n      \"endTime\": \"2026-10-14T19:38:30.41816485Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e22f032f-fd1f-4f7a-a0f0-79ca0d681ff5\n            correlation ID: 40e10400-2a72-4df0-a76e-c12b12d5889c\n            client request ID: 5416d7ee-a6cc-4125-9004-a1fe5836d05b\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-mxm2nz\" with version 4.21.36 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e81b57e6100\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-mxm2nz: failed waiting for cluster=\"cluster-zstream-4-21-mxm2nz\" in resourcegroup=\"rg-zstream-upgrade-4-21-9f22s7ghtq4x\" to finish creating: GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e22f032f-fd1f-4f7a-a0f0-79ca0d681ff5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e22f032f-fd1f-4f7a-a0f0-79ca0d681ff5\",\n      \"name\": \"e22f032f-fd1f-4f7a-a0f0-79ca0d681ff5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T19:09:23.643995805Z\",\n      \"endTime\": \"2026-10-14T19:38:30.41816485Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e22f032f-fd1f-4f7a-a0f0-79ca0d681ff5\n            correlation ID: 40e10400-2a72-4df0-a76e-c12b12d5889c\n            client request ID: 5416d7ee-a6cc-4125-9004-a1fe5836d05b\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110436010226618368","occurred_at":"2026-10-14T18:22:06Z","signature_id":"143def6409b35b1c7d757163265b805bf1b1688f374686fad28f34887ebc3acc","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110436010226618368","occurred_at":"2026-10-14T18:22:06Z","signature_id":"24aeb68202ddb418951a2358babdeb1022f31ae3504b6e8ebf13c0be83fcf229","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110436010226618368","occurred_at":"2026-10-14T18:22:06Z","signature_id":"6f9cf8d30ba68d9f16ced08b0ca5a22bded54c97f83f0a19c451ec450b4450ad","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110436010226618368","occurred_at":"2026-10-14T18:22:06Z","signature_id":"9de95f427e1e691bee870aa47fe10797350a1d239b34784b0123ece1c0d37502","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110436010226618368","occurred_at":"2026-10-14T18:22:06Z","signature_id":"ac1f00c956f42dd7a153687ebb08193db10a4521feb9fe5150ebbc130419e571","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"179b78cf112d379960e54d9d4b5b459a8f1cc6bfa3bbb31725a79978e9184c64","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"4b731517316ce3a5808ec2ba5ecb34ecc20952fffb5e19de022a57c410f4e2f4","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"9ed4cd1b6f3c34dbaf9c5e754cc24b690545748ca53215644722020cca18bdae","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"e7075774c82d3677e833d8f6c5fca557dda0e4dfa59ef31282500a86e593da0b","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-z6jkddr7rndp\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":5,"anchor_occurrences":7,"after_last_push_count":1,"trend_counts":[0,5,2,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":123,"prior_last_seen_at":"2026-10-15T05:42:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x38b9840538e0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8w5l7nl2mg2p\" to finish creating: GET https://rp.j7178624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f20f693c-30b3-4028-bef9-79a61131ae05\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f20f693c-30b3-4028-bef9-79a61131ae05\",\n      \"name\": \"f20f693c-30b3-4028-bef9-79a61131ae05\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T06:40:48.977059858Z\",\n      \"endTime\": \"2026-10-15T07:09:53.223490028Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine i3x0k0f4h4m3h2m-np-delete-main-97ksv-4fhzs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8w5l7nl2mg2p--managed/providers/Microsoft.Compute/virtualMachines/i3x0k0f4h4m3h2m-np-delete-main-97ksv-4fhzs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i3x0k0f4h4m3h2m-np-delete-main-97ksv-4fhzs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8w5l7nl2mg2p--managed/providers/Microsoft.Compute/virtualMachines/i3x0k0f4h4m3h2m-np-delete-main-97ksv-4fhzs to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7178624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f20f693c-30b3-4028-bef9-79a61131ae05\n            correlation ID: c5b64ec2-f2e3-4ea2-80c9-40d2bea7a8eb\n            client request ID: 8db26239-6ff9-4e91-938f-c7c27b5dc1e1\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x38b9840538e0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8w5l7nl2mg2p\" to finish creating: GET https://rp.j7178624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f20f693c-30b3-4028-bef9-79a61131ae05\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f20f693c-30b3-4028-bef9-79a61131ae05\",\n      \"name\": \"f20f693c-30b3-4028-bef9-79a61131ae05\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T06:40:48.977059858Z\",\n      \"endTime\": \"2026-10-15T07:09:53.223490028Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine i3x0k0f4h4m3h2m-np-delete-main-97ksv-4fhzs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8w5l7nl2mg2p--managed/providers/Microsoft.Compute/virtualMachines/i3x0k0f4h4m3h2m-np-delete-main-97ksv-4fhzs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i3x0k0f4h4m3h2m-np-delete-main-97ksv-4fhzs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8w5l7nl2mg2p--managed/providers/Microsoft.Compute/virtualMachines/i3x0k0f4h4m3h2m-np-delete-main-97ksv-4fhzs to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7178624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f20f693c-30b3-4028-bef9-79a61131ae05\n            correlation ID: c5b64ec2-f2e3-4ea2-80c9-40d2bea7a8eb\n            client request ID: 8db26239-6ff9-4e91-938f-c7c27b5dc1e1","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x25270548a240\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-gzks5p7t2hk9\" to finish creating: GET https://rp.j5852928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83ccd50a-e6ff-4e36-bbaf-b33a003914e2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83ccd50a-e6ff-4e36-bbaf-b33a003914e2\",\n      \"name\": \"83ccd50a-e6ff-4e36-bbaf-b33a003914e2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T03:39:14.825885335Z\",\n      \"endTime\": \"2026-10-15T04:08:21.442426792Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l9m9u8n7x8v1d4q-np-deletable-7glhq-wncf7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-gzks5p7t2hk9--managed/providers/Microsoft.Compute/virtualMachines/l9m9u8n7x8v1d4q-np-deletable-7glhq-wncf7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l9m9u8n7x8v1d4q-np-deletable-7glhq-wncf7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-gzks5p7t2hk9--managed/providers/Microsoft.Compute/virtualMachines/l9m9u8n7x8v1d4q-np-deletable-7glhq-wncf7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5852928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83ccd50a-e6ff-4e36-bbaf-b33a003914e2\n            correlation ID: 23f82d10-9ae2-45ee-9e11-92c0cf8042ec\n            client request ID: 53104264-d81f-472b-a94d-ed6c3c435504\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x25270548a240\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-gzks5p7t2hk9\" to finish creating: GET https://rp.j5852928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83ccd50a-e6ff-4e36-bbaf-b33a003914e2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83ccd50a-e6ff-4e36-bbaf-b33a003914e2\",\n      \"name\": \"83ccd50a-e6ff-4e36-bbaf-b33a003914e2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T03:39:14.825885335Z\",\n      \"endTime\": \"2026-10-15T04:08:21.442426792Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l9m9u8n7x8v1d4q-np-deletable-7glhq-wncf7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-gzks5p7t2hk9--managed/providers/Microsoft.Compute/virtualMachines/l9m9u8n7x8v1d4q-np-deletable-7glhq-wncf7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l9m9u8n7x8v1d4q-np-deletable-7glhq-wncf7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-gzks5p7t2hk9--managed/providers/Microsoft.Compute/virtualMachines/l9m9u8n7x8v1d4q-np-deletable-7glhq-wncf7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5852928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83ccd50a-e6ff-4e36-bbaf-b33a003914e2\n            correlation ID: 23f82d10-9ae2-45ee-9e11-92c0cf8042ec\n            client request ID: 53104264-d81f-472b-a94d-ed6c3c435504","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3ad81c033ca0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-gsxwfjg2wp8q\" to finish creating: GET https://rp.j6056960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b803943b-9e39-4dba-aaa8-6afe2e02cb47\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b803943b-9e39-4dba-aaa8-6afe2e02cb47\",\n      \"name\": \"b803943b-9e39-4dba-aaa8-6afe2e02cb47\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T22:01:54.508607559Z\",\n      \"endTime\": \"2026-10-14T22:30:54.403314647Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u0w4f2e8s8w0a8w-np-delete-main-tbj7t-gshqg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-gsxwfjg2wp8q--managed/providers/Microsoft.Compute/virtualMachines/u0w4f2e8s8w0a8w-np-delete-main-tbj7t-gshqg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u0w4f2e8s8w0a8w-np-delete-main-tbj7t-gshqg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-gsxwfjg2wp8q--managed/providers/Microsoft.Compute/virtualMachines/u0w4f2e8s8w0a8w-np-delete-main-tbj7t-gshqg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6056960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b803943b-9e39-4dba-aaa8-6afe2e02cb47\n            correlation ID: 988bdfac-593d-495e-9f75-b7aaa11925ed\n            client request ID: 251c1e65-e123-4756-a69e-5f6923040f9c\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3ad81c033ca0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-gsxwfjg2wp8q\" to finish creating: GET https://rp.j6056960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b803943b-9e39-4dba-aaa8-6afe2e02cb47\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b803943b-9e39-4dba-aaa8-6afe2e02cb47\",\n      \"name\": \"b803943b-9e39-4dba-aaa8-6afe2e02cb47\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T22:01:54.508607559Z\",\n      \"endTime\": \"2026-10-14T22:30:54.403314647Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u0w4f2e8s8w0a8w-np-delete-main-tbj7t-gshqg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-gsxwfjg2wp8q--managed/providers/Microsoft.Compute/virtualMachines/u0w4f2e8s8w0a8w-np-delete-main-tbj7t-gshqg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u0w4f2e8s8w0a8w-np-delete-main-tbj7t-gshqg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-gsxwfjg2wp8q--managed/providers/Microsoft.Compute/virtualMachines/u0w4f2e8s8w0a8w-np-delete-main-tbj7t-gshqg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6056960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b803943b-9e39-4dba-aaa8-6afe2e02cb47\n            correlation ID: 988bdfac-593d-495e-9f75-b7aaa11925ed\n            client request ID: 251c1e65-e123-4756-a69e-5f6923040f9c"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7487/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110607153617178624","occurred_at":"2026-10-15T05:42:13Z","signature_id":"97bfd8c685ed25785b98d0c464723104c6310fb126eedbfee4754118c08c6c7e","pr_number":7487,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558373865852928","occurred_at":"2026-10-15T02:28:23Z","signature_id":"6a6340d98166407ddced8d99afdbb8460d70e92d4881d4189e7a029a2ec7ed6f","pr_number":7524,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7538/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110474243606056960","occurred_at":"2026-10-14T20:54:04Z","signature_id":"2edb65bb26c1fcf13b3d6493ff98a75094606084998e2774dd1fdd5d9eea40d3","pr_number":7538,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110436010226618368","occurred_at":"2026-10-14T18:22:06Z","signature_id":"2bcbcddc1129669cc16a652effa6b618e89105ac18bfc5c7045a63681a784df7","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7525/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110338041116626944","occurred_at":"2026-10-14T11:52:51Z","signature_id":"ef7bc72ecbccb9636abbff125ef303658b2a72e4790bc2dc66060294d79a7d66","pr_number":7525,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110305876723109888","occurred_at":"2026-10-14T09:44:59Z","signature_id":"0f15a710eff8933fff6957af6014aa8dd4f7920e26e97a6c196fb7ab83a7a410","pr_number":7497,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110168566228586496","occurred_at":"2026-10-14T00:39:25Z","signature_id":"ddbe6c18bc9334463938901bbf610b5d5f5e544a53db6228b02093b131657e11","pr_number":7248,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":3.5714285714285716,"anchor_occurrences":5,"after_last_push_count":8,"trend_counts":[0,5,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":27,"prior_last_seen_at":"2026-10-14T14:05:09Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1fd36b60420\u003e: \n    DNS for route host agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1fd36b60400\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1fd36b60420\u003e: \n    DNS for route host agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1fd36b60400\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x14a4f3922100\u003e: \n    DNS for route host agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x14a4f39220e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x14a4f3922100\u003e: \n    DNS for route host agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x14a4f39220e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x4eb350769e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x4eb350769c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x4eb350769e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x4eb350769c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110371366967447552","occurred_at":"2026-10-14T14:05:09Z","signature_id":"9e22c62935cd79e964f918ea959f94f3511d0bddaffe8d45ff361c9c1abb7ca7","pr_number":7519,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7430/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110316641895059456","occurred_at":"2026-10-14T10:27:49Z","signature_id":"ed1ed5ed2432b0c9c13a6ec98fa3bb36db707b1b3b979c88ae7437a7a6e87e63","pr_number":7430,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110275068176109568","occurred_at":"2026-10-14T07:42:37Z","signature_id":"815c2d271041727d3fd2c78730b42d4a5396d584aa19c429bcd82b945077a214","pr_number":7492,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7515/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110253677582225408","occurred_at":"2026-10-14T06:17:37Z","signature_id":"f9e54dc2e97245d91f304a9f6c2ad9e348ba515da09f1b844adfe4d35dd65d2f","pr_number":7515,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110168566228586496","occurred_at":"2026-10-14T00:39:25Z","signature_id":"ad240022cb81ab8363b3db81cf92564f82d7d9330d100781ceec147c73181eea","pr_number":7248,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":3.5714285714285716,"anchor_occurrences":5,"after_last_push_count":25,"trend_counts":[6,4,1,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":94,"prior_last_seen_at":"2026-10-15T04:29:39Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":5}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-15T05:57:23Z\n  Ended:       2026-10-15T06:03:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8670976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tf181j4rusq6pkjnivolrv9gud9pk5v-x9q9a7d4u8q5f3x\", pod=\"ignition-server-6cfc56b87-qjx7r\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tf181j4rusq6pkjnivolrv9gud9pk5v-x9q9a7d4u8q5f3x/ignition-server-6cfc56b87-qjx7r has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-14T20:10:46Z\n  Ended:       2026-10-14T20:41:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3670144-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tep2lb9i2eso8etu810kd5rk9tkuprr-a1f8l5v2m8b4i0a\", pod=\"router-84bfd8949d-rj9jm\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tep2lb9i2eso8etu810kd5rk9tkuprr-a1f8l5v2m8b4i0a/router-84bfd8949d-rj9jm has been in a non-ready state for longer than 5 minutes.","alert fired 6 time(s) (3 unknown, 3 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-14T14:05:08Z\n  Ended:       2026-10-14T14:16:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2105600-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tejjhusc5envac7crsmjjhhgio0a4aj-k5o0u8o1m0i8z8z\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tejjhusc5envac7crsmjjhhgio0a4aj-k5o0u8o1m0i8z8z/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-14T14:05:09Z\n  Ended:       2026-10-14T14:16:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2105600-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tejjc08ajbv33a72bs84k5higrqkmc1-private-kas\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tejjc08ajbv33a72bs84k5higrqkmc1-private-kas/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-14T14:05:09Z\n  Ended:       2026-10-14T14:16:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2105600-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tejjl9d0k8tfj5mg7df7i9dvhecig15-d7v8f1i0h1a6r7x\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tejjl9d0k8tfj5mg7df7i9dvhecig15-d7v8f1i0h1a6r7x/etcd-1 has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110588894188670976","occurred_at":"2026-10-15T04:29:39Z","signature_id":"1b16c2939746e07666536f5967563ec30a93299a662662a970a43f500af9b120","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110447856593670144","occurred_at":"2026-10-14T19:09:11Z","signature_id":"92d0db446a1dc21a2aad36ec9d8dada9e5f4b84c78fc10253e30a3cae65b1b1c","pr_number":7529,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7522/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110350863582105600","occurred_at":"2026-10-14T12:43:47Z","signature_id":"8e24ab0f93da5f4c9e06cf1e6c9a5dad864de5524f0ff38944de964d0d11c1d0","pr_number":7522,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110251822873579520","occurred_at":"2026-10-14T06:10:15Z","signature_id":"0737293351a530d5699a84b32dfec7c3c5268b0328193be453aec2ef79c7ae95","pr_number":7504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110199338117369856","occurred_at":"2026-10-14T02:41:42Z","signature_id":"17761d523d7af03109fc74ba2a7283ce86a08f29c2f5ee6925a34b270860af93","pr_number":7495,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"56d972e76712743c510249b7bfaf8d1cef7f97d62d33d6258594d264ca2ff6b2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,5,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-14T02:08:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-8fmvrw\" to 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x189d56536760\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-8fmvrw\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-m4rkshgrgs9x\" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912\",\n      \"name\": \"008de110-0fc7-4cfb-b1a8-16dd774cc912\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T03:16:19.986510176Z\",\n      \"endTime\": \"2026-10-14T04:00:20.242906036Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912\n            correlation ID: edcafa24-51c5-48f1-9ce5-cdc56c9cfe21\n            client request ID: d7ffd994-e6d8-430b-b52c-c25abf00d2d4\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-8fmvrw\" to 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x189d56536760\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-8fmvrw\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-m4rkshgrgs9x\" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912\",\n      \"name\": \"008de110-0fc7-4cfb-b1a8-16dd774cc912\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T03:16:19.986510176Z\",\n      \"endTime\": \"2026-10-14T04:00:20.242906036Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912\n            correlation ID: edcafa24-51c5-48f1-9ce5-cdc56c9cfe21\n            client request ID: d7ffd994-e6d8-430b-b52c-c25abf00d2d4\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-v9gdsd\" to 5.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0x89c8a7b4520\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-v9gdsd\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-lrtt9r887h59\" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0\",\n      \"name\": \"149ad663-a4d9-4ebd-9101-8c188eed65d0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T03:29:27.04612928Z\",\n      \"endTime\": \"2026-10-14T04:13:34.922970159Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0\n            correlation ID: b89bc015-38c4-4b8f-8cf6-fd7d29e794e7\n            client request ID: c975ef93-0023-4a12-b6b6-8dfd3c0b6abf\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-v9gdsd\" to 5.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0x89c8a7b4520\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-v9gdsd\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-lrtt9r887h59\" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0\",\n      \"name\": \"149ad663-a4d9-4ebd-9101-8c188eed65d0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T03:29:27.04612928Z\",\n      \"endTime\": \"2026-10-14T04:13:34.922970159Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0\n            correlation ID: b89bc015-38c4-4b8f-8cf6-fd7d29e794e7\n            client request ID: c975ef93-0023-4a12-b6b6-8dfd3c0b6abf\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-trt4r8\" to 4.23\nUnexpected error:\n    \u003c*fmt.wrapError | 0xbc2497a0a40\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-trt4r8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-nm455m5dz4rg\" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3\",\n      \"name\": \"ab2c099b-e08d-48bf-8615-536073622db3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T03:17:19.784745543Z\",\n      \"endTime\": \"2026-10-14T04:01:20.236478176Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3\n            correlation ID: b6e77c38-1227-46e6-9bc5-a4baba263145\n            client request ID: fcf783cc-37c0-4ef5-8f68-49e5b907b57c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-trt4r8\" to 4.23\nUnexpected error:\n    \u003c*fmt.wrapError | 0xbc2497a0a40\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-trt4r8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-nm455m5dz4rg\" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3\",\n      \"name\": \"ab2c099b-e08d-48bf-8615-536073622db3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T03:17:19.784745543Z\",\n      \"endTime\": \"2026-10-14T04:01:20.236478176Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3\n            correlation ID: b6e77c38-1227-46e6-9bc5-a4baba263145\n            client request ID: fcf783cc-37c0-4ef5-8f68-49e5b907b57c\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110191032388816896","occurred_at":"2026-10-14T02:08:42Z","signature_id":"5d250e50a264778942c74b3ce0da1834bd17192a108f576c77963d2bd5104c98","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110191032388816896","occurred_at":"2026-10-14T02:08:42Z","signature_id":"850a794b118cd701611798b45a5bcb03f88a9401ed189a5b8f264b754f82d23a","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110191032388816896","occurred_at":"2026-10-14T02:08:42Z","signature_id":"8825f796a0bcda6f47b5a66cc92693cc86f0c5ca1b9550e8c9c028e39821e838","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110191032388816896","occurred_at":"2026-10-14T02:08:42Z","signature_id":"93eac86ad49dfa02d26b8b40788a25a3f63d761987ff78fe67ff401dea4135e9","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110191032388816896","occurred_at":"2026-10-14T02:08:42Z","signature_id":"e89a0c8726bcc2e849495d81caf51843d12d66f11e5cc3df7b6a10e2304b7d01","pr_number":6968,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":2.857142857142857,"anchor_occurrences":4,"after_last_push_count":7,"trend_counts":[1,4,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":15,"prior_last_seen_at":"2026-10-14T20:12:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5bcdb5a5440\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-5gxqdd8p2qbn\" to finish creating: GET https://rp.j3055232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9da300d2-61ca-49b6-8cec-c2ecaffe701e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9da300d2-61ca-49b6-8cec-c2ecaffe701e\",\n      \"name\": \"9da300d2-61ca-49b6-8cec-c2ecaffe701e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T21:26:38.348499296Z\",\n      \"endTime\": \"2026-10-14T21:55:39.3330542Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3055232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9da300d2-61ca-49b6-8cec-c2ecaffe701e\n            correlation ID: 7d23b8d5-7dba-484d-af3a-9becf223a3c1\n            client request ID: 66e7e251-c1da-4383-aef0-9b31e92742c7\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5bcdb5a5440\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-5gxqdd8p2qbn\" to finish creating: GET https://rp.j3055232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9da300d2-61ca-49b6-8cec-c2ecaffe701e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9da300d2-61ca-49b6-8cec-c2ecaffe701e\",\n      \"name\": \"9da300d2-61ca-49b6-8cec-c2ecaffe701e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T21:26:38.348499296Z\",\n      \"endTime\": \"2026-10-14T21:55:39.3330542Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3055232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9da300d2-61ca-49b6-8cec-c2ecaffe701e\n            correlation ID: 7d23b8d5-7dba-484d-af3a-9becf223a3c1\n            client request ID: 66e7e251-c1da-4383-aef0-9b31e92742c7\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vtxdl7\" for cluster \"cluster-candidate-5-0-vtxdl7\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x23d38cc9d0c0\u003e: \n    failed to create NodePool np-vtxdl7: failed waiting for nodepool=\"np-vtxdl7\" for cluster \"cluster-candidate-5-0-vtxdl7\" in resourcegroup=\"rg-candidate-5-0-llmp2gg5r5wc\" to finish creating: GET https://rp.j0018432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/53b3d6e3-5baf-4a04-b762-6c77f3e9c4a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/53b3d6e3-5baf-4a04-b762-6c77f3e9c4a2\",\n      \"name\": \"53b3d6e3-5baf-4a04-b762-6c77f3e9c4a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T15:40:21.526109364Z\",\n      \"endTime\": \"2026-10-14T16:09:23.214850022Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0018432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/53b3d6e3-5baf-4a04-b762-6c77f3e9c4a2\n            correlation ID: 1427370e-268a-4518-bc82-b69439593955\n            client request ID: 074febab-8e11-4f6f-b6d6-be4d97f6fe52\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vtxdl7\" for cluster \"cluster-candidate-5-0-vtxdl7\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x23d38cc9d0c0\u003e: \n    failed to create NodePool np-vtxdl7: failed waiting for nodepool=\"np-vtxdl7\" for cluster \"cluster-candidate-5-0-vtxdl7\" in resourcegroup=\"rg-candidate-5-0-llmp2gg5r5wc\" to finish creating: GET https://rp.j0018432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/53b3d6e3-5baf-4a04-b762-6c77f3e9c4a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/53b3d6e3-5baf-4a04-b762-6c77f3e9c4a2\",\n      \"name\": \"53b3d6e3-5baf-4a04-b762-6c77f3e9c4a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T15:40:21.526109364Z\",\n      \"endTime\": \"2026-10-14T16:09:23.214850022Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0018432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/53b3d6e3-5baf-4a04-b762-6c77f3e9c4a2\n            correlation ID: 1427370e-268a-4518-bc82-b69439593955\n            client request ID: 074febab-8e11-4f6f-b6d6-be4d97f6fe52\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xf6a2f24c800\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-lpcs9lxsfkb6\" to finish creating: GET https://rp.j5913088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7476d679-0544-4ea6-98bc-8369b57375b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7476d679-0544-4ea6-98bc-8369b57375b6\",\n      \"name\": \"7476d679-0544-4ea6-98bc-8369b57375b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T14:43:09.62497089Z\",\n      \"endTime\": \"2026-10-14T15:12:14.517634212Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5913088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7476d679-0544-4ea6-98bc-8369b57375b6\n            correlation ID: b2aadf48-cb5e-49df-828e-ed7c131078ce\n            client request ID: a92178cd-8f13-4632-bde1-e16a3dc65cc2\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xf6a2f24c800\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-lpcs9lxsfkb6\" to finish creating: GET https://rp.j5913088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7476d679-0544-4ea6-98bc-8369b57375b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7476d679-0544-4ea6-98bc-8369b57375b6\",\n      \"name\": \"7476d679-0544-4ea6-98bc-8369b57375b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T14:43:09.62497089Z\",\n      \"endTime\": \"2026-10-14T15:12:14.517634212Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5913088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7476d679-0544-4ea6-98bc-8369b57375b6\n            correlation ID: b2aadf48-cb5e-49df-828e-ed7c131078ce\n            client request ID: a92178cd-8f13-4632-bde1-e16a3dc65cc2\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6866/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110463708953055232","occurred_at":"2026-10-14T20:12:13Z","signature_id":"cecf631ae361f8a51d4b72842068b68b69f80342291c1cbf1990a183f6131c1e","pr_number":6866,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110377553440018432","occurred_at":"2026-10-14T14:29:52Z","signature_id":"37191323f61b58a1822c2e181e6491d477ddd25ec6e827fd594606032d08cfcf","pr_number":7529,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110361307805913088","occurred_at":"2026-10-14T13:25:18Z","signature_id":"0aa02a14101484db576124bfb9226ae693d8ca3ab3cc3bef546f2e12d693deba","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110334779541950464","occurred_at":"2026-10-14T11:39:53Z","signature_id":"2fcc5bf330faefb96ced672e9367aef3eab81cd45e1be28037ce9e2ea61effd3","pr_number":7519,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"0ab80c2f2404154d860a57027c4c22b196a560eb7d17aca0414b079cdbdcf0e8","failure_pattern":"VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s","search_query":"VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.22","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,4,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T12:02:45Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x26b7f5580860\u003e: \n    VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\n    {\n        s: \"VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x26b7f5580860\u003e: \n    VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\n    {\n        s: \"VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x5b6d94696d0\u003e: \n    VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\n    {\n        s: \"VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x5b6d94696d0\u003e: \n    VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\n    {\n        s: \"VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x362088f5cd60\u003e: \n    VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\n    {\n        s: \"VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x362088f5cd60\u003e: \n    VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\n    {\n        s: \"VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"12a803c93bfc31c149c19e19babb26d0b3c6504666182198eaa16e7612f760e8","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"37bcf36d54f44bfeee035bd8567e75aee5c0283f942d5d8f119f3f70bcf86f25","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"43c0a9246f132ce547efe4ef404ff816fd70dec0983aadabc64c9b64714c506a","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"5a3d8ec5914654258d035b53c13a9219b5025b4a9e88fc022e31972f06ab9a6c","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":2.142857142857143,"anchor_occurrences":3,"after_last_push_count":10,"trend_counts":[6,3,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":35,"prior_last_seen_at":"2026-10-14T22:15:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0x24524eb9ea50\u003e: \n    GET https://rp.j0214528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/812fe66b-c9fc-46f9-8d46-bf8a0cd5068b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/812fe66b-c9fc-46f9-8d46-bf8a0cd5068b\",\n      \"name\": \"812fe66b-c9fc-46f9-8d46-bf8a0cd5068b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:04:33.551295422Z\",\n      \"endTime\": \"2026-10-14T23:33:41.530060174Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0214528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/812fe66b-c9fc-46f9-8d46-bf8a0cd5068b\n            correlation ID: d14d5694-03de-4ac8-b587-4d0e35762d84\n            client request ID: 5cfd8941-3a60-4ebe-8bc6-a659901f1339\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0x24524eb9ea50\u003e: \n    GET https://rp.j0214528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/812fe66b-c9fc-46f9-8d46-bf8a0cd5068b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/812fe66b-c9fc-46f9-8d46-bf8a0cd5068b\",\n      \"name\": \"812fe66b-c9fc-46f9-8d46-bf8a0cd5068b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:04:33.551295422Z\",\n      \"endTime\": \"2026-10-14T23:33:41.530060174Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0214528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/812fe66b-c9fc-46f9-8d46-bf8a0cd5068b\n            correlation ID: d14d5694-03de-4ac8-b587-4d0e35762d84\n            client request ID: 5cfd8941-3a60-4ebe-8bc6-a659901f1339\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0x225a34843540\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5rp2rtlm9fj9\" to finish creating: GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/680a3372-b50b-4c21-a07b-0256c4145619\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/680a3372-b50b-4c21-a07b-0256c4145619\",\n      \"name\": \"680a3372-b50b-4c21-a07b-0256c4145619\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T09:49:54.805487688Z\",\n      \"endTime\": \"2026-10-14T10:18:57.194996434Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/680a3372-b50b-4c21-a07b-0256c4145619\n            correlation ID: 5895723a-3ff0-431a-a927-5b6e194fc741\n            client request ID: 350b7a37-c335-451c-b2e9-99a3b665afa1\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0x225a34843540\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5rp2rtlm9fj9\" to finish creating: GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/680a3372-b50b-4c21-a07b-0256c4145619\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/680a3372-b50b-4c21-a07b-0256c4145619\",\n      \"name\": \"680a3372-b50b-4c21-a07b-0256c4145619\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T09:49:54.805487688Z\",\n      \"endTime\": \"2026-10-14T10:18:57.194996434Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/680a3372-b50b-4c21-a07b-0256c4145619\n            correlation ID: 5895723a-3ff0-431a-a927-5b6e194fc741\n            client request ID: 350b7a37-c335-451c-b2e9-99a3b665afa1\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3e322e6e67c0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-7rswnntg2mwm\" to finish creating: GET https://rp.j2666368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eeb29541-e4cc-419e-8a9e-17c262940b71\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eeb29541-e4cc-419e-8a9e-17c262940b71\",\n      \"name\": \"eeb29541-e4cc-419e-8a9e-17c262940b71\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T05:02:26.935820049Z\",\n      \"endTime\": \"2026-10-14T05:31:33.678218143Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2666368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eeb29541-e4cc-419e-8a9e-17c262940b71\n            correlation ID: 91b1a5de-a9bc-4e6e-ad6d-e45a7cb1831a\n            client request ID: 4a6ae7e3-e39d-4cda-bb39-3082e7e48f98\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3e322e6e67c0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-7rswnntg2mwm\" to finish creating: GET https://rp.j2666368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eeb29541-e4cc-419e-8a9e-17c262940b71\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eeb29541-e4cc-419e-8a9e-17c262940b71\",\n      \"name\": \"eeb29541-e4cc-419e-8a9e-17c262940b71\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T05:02:26.935820049Z\",\n      \"endTime\": \"2026-10-14T05:31:33.678218143Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2666368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eeb29541-e4cc-419e-8a9e-17c262940b71\n            correlation ID: 91b1a5de-a9bc-4e6e-ad6d-e45a7cb1831a\n            client request ID: 4a6ae7e3-e39d-4cda-bb39-3082e7e48f98\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110494690280214528","occurred_at":"2026-10-14T22:15:16Z","signature_id":"ed9fcd46de5fd368c22dffbc0a6841cff99957b615f7c90f7722dcf43f30b6fb","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110291728471691264","occurred_at":"2026-10-14T08:48:48Z","signature_id":"801661c2c5d4111b3b2777864164064d45ed90a83f09dd5139dcd40652bab0e6","pr_number":7500,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110213919732666368","occurred_at":"2026-10-14T03:39:38Z","signature_id":"bccfe50993c9da8f2e5a003a505f4895a9e3eda1fc47df0e84d9a7dfd246b05a","pr_number":7509,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":2.142857142857143,"anchor_occurrences":3,"after_last_push_count":4,"trend_counts":[0,2,1,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":12,"prior_last_seen_at":"2026-10-15T05:43:27Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x1808f1276588\u003e: \n    resource group \"des-encrypt-fr8cqml7zsjj\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-fr8cqml7zsjj\" to finish deleting: GET https://rp.j7707648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d46ca823-6a99-453f-a91b-c390aed9f889\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d46ca823-6a99-453f-a91b-c390aed9f889\",\n      \"name\": \"d46ca823-6a99-453f-a91b-c390aed9f889\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T06:45:35.515831022Z\",\n      \"endTime\": \"2026-10-15T07:09:37.554161836Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tf236h3grf5nh5cka2klv381nroha64 still exists (deletion dispatched at 2026-10-15T06:45:36Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7707648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d46ca823-6a99-453f-a91b-c390aed9f889\n            correlation ID: 9dd61a19-eb6b-42e3-962a-725be28358e3\n            client request ID: 0e89fe2d-1574-456d-9c74-1a2277b25a8d\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x1808f1276588\u003e: \n    resource group \"des-encrypt-fr8cqml7zsjj\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-fr8cqml7zsjj\" to finish deleting: GET https://rp.j7707648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d46ca823-6a99-453f-a91b-c390aed9f889\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d46ca823-6a99-453f-a91b-c390aed9f889\",\n      \"name\": \"d46ca823-6a99-453f-a91b-c390aed9f889\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T06:45:35.515831022Z\",\n      \"endTime\": \"2026-10-15T07:09:37.554161836Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tf236h3grf5nh5cka2klv381nroha64 still exists (deletion dispatched at 2026-10-15T06:45:36Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7707648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d46ca823-6a99-453f-a91b-c390aed9f889\n            correlation ID: 9dd61a19-eb6b-42e3-962a-725be28358e3\n            client request ID: 0e89fe2d-1574-456d-9c74-1a2277b25a8d\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0xba275880060\u003e: \n    resource group \"customer-rg-xns69mfwd2hq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-xns69mfwd2hq\" to finish deleting: GET https://rp.j6205824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3176ad81-fee9-4cf2-8688-8a1c24844387\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3176ad81-fee9-4cf2-8688-8a1c24844387\",\n      \"name\": \"3176ad81-fee9-4cf2-8688-8a1c24844387\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T18:01:05.468018625Z\",\n      \"endTime\": \"2026-10-14T18:25:10.474406797Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ten52g4l2k3e0ijcet6akeh14ne2nbv still exists (deletion dispatched at 2026-10-14T18:01:06Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6205824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3176ad81-fee9-4cf2-8688-8a1c24844387\n            correlation ID: fdc30364-8ed7-4835-a286-f7a091278652\n            client request ID: a6850f11-48fa-4616-8112-2b5b344c479a\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0xba275880060\u003e: \n    resource group \"customer-rg-xns69mfwd2hq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-xns69mfwd2hq\" to finish deleting: GET https://rp.j6205824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3176ad81-fee9-4cf2-8688-8a1c24844387\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3176ad81-fee9-4cf2-8688-8a1c24844387\",\n      \"name\": \"3176ad81-fee9-4cf2-8688-8a1c24844387\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T18:01:05.468018625Z\",\n      \"endTime\": \"2026-10-14T18:25:10.474406797Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ten52g4l2k3e0ijcet6akeh14ne2nbv still exists (deletion dispatched at 2026-10-14T18:01:06Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6205824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3176ad81-fee9-4cf2-8688-8a1c24844387\n            correlation ID: fdc30364-8ed7-4835-a286-f7a091278652\n            client request ID: a6850f11-48fa-4616-8112-2b5b344c479a\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x29ce27c860f0\u003e: \n    resource group \"ea-list-7knxqnrqn2r4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listrj8r2m\" in resourcegroup=\"ea-list-7knxqnrqn2r4\" to finish deleting: GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d6add57-b9e0-49e9-82fe-b8f41bea59ca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d6add57-b9e0-49e9-82fe-b8f41bea59ca\",\n      \"name\": \"7d6add57-b9e0-49e9-82fe-b8f41bea59ca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T13:07:39.341755996Z\",\n      \"endTime\": \"2026-10-14T13:31:39.648045292Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2teirqf5984h2i8hr97b4hmu155vmka1 still exists (deletion dispatched at 2026-10-14T13:07:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d6add57-b9e0-49e9-82fe-b8f41bea59ca\n            correlation ID: a29d8857-e9bf-4a4f-bec4-d3e72ff8ea78\n            client request ID: 11bba4e8-0934-463b-98c2-cdf775749155\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x29ce27c860f0\u003e: \n    resource group \"ea-list-7knxqnrqn2r4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listrj8r2m\" in resourcegroup=\"ea-list-7knxqnrqn2r4\" to finish deleting: GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d6add57-b9e0-49e9-82fe-b8f41bea59ca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d6add57-b9e0-49e9-82fe-b8f41bea59ca\",\n      \"name\": \"7d6add57-b9e0-49e9-82fe-b8f41bea59ca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T13:07:39.341755996Z\",\n      \"endTime\": \"2026-10-14T13:31:39.648045292Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2teirqf5984h2i8hr97b4hmu155vmka1 still exists (deletion dispatched at 2026-10-14T13:07:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d6add57-b9e0-49e9-82fe-b8f41bea59ca\n            correlation ID: a29d8857-e9bf-4a4f-bec4-d3e72ff8ea78\n            client request ID: 11bba4e8-0934-463b-98c2-cdf775749155\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110607466327707648","occurred_at":"2026-10-15T05:43:27Z","signature_id":"b7fa26eab1a720d5e3645d12c57cf4f2ad36521990ec6cafd9cf46aa09d22a60","pr_number":7509,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7301/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110412681776205824","occurred_at":"2026-10-14T16:49:27Z","signature_id":"805ce9687446a53482c8cbf283a80c102b3c36ffbdef6a91f5aa6520b895724e","pr_number":7301,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"1c56e7df692c6920b8c980b71c8374582ba0ef9f87b43803546d8d8bbfe1ae3b","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":2.142857142857143,"anchor_occurrences":3,"after_last_push_count":9,"trend_counts":[2,2,1,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":40,"prior_last_seen_at":"2026-10-15T02:28:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":3}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T04:10:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7932288-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.98:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7932288/providers/microsoft.monitor/accounts/hcps-j7932288\", namespace=\"ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w\", pod=\"cloud-network-config-controller-5dbf455969-n498c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"724a7794-4308-471d-8b59-9caf89cd21b6\"\n  Description: Pod ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w/cloud-network-config-controller-5dbf455969-n498c (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T04:10:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7932288-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.98:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7932288/providers/microsoft.monitor/accounts/hcps-j7932288\", namespace=\"ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w\", pod=\"cloud-network-config-controller-5dbf455969-n498c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"724a7794-4308-471d-8b59-9caf89cd21b6\"\n  Description: Pod ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w/cloud-network-config-controller-5dbf455969-n498c (controller) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-14T23:26:27Z\n  Ended:       2026-10-14T23:56:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0214528-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0214528/providers/microsoft.monitor/accounts/hcps-j0214528\", namespace=\"ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e7a6870f-14b4-45e0-9c19-fce7f03813dc\"\n  Description: Pod ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-14T23:26:27Z\n  Ended:       2026-10-14T23:56:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0214528-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0214528/providers/microsoft.monitor/accounts/hcps-j0214528\", namespace=\"ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e7a6870f-14b4-45e0-9c19-fce7f03813dc\"\n  Description: Pod ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-14T10:12:41Z\n  Ended:       2026-10-14T10:40:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1691264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.235:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1691264/providers/microsoft.monitor/accounts/hcps-j1691264\", namespace=\"ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"59d27077-4da5-493d-b6e6-b015a3abbc8a\"\n  Description: Pod ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-14T10:12:41Z\n  Ended:       2026-10-14T10:41:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1691264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.235:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1691264/providers/microsoft.monitor/accounts/hcps-j1691264\", namespace=\"ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"59d27077-4da5-493d-b6e6-b015a3abbc8a\"\n  Description: Pod ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558444497932288","occurred_at":"2026-10-15T02:28:40Z","signature_id":"ba8d8f3740082be57570aa73aa1e24accf3cf4552b7d0a87a0841840a1f288ed","pr_number":7492,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110494690280214528","occurred_at":"2026-10-14T22:15:16Z","signature_id":"4d0f56a4ffddc70a5d6865e72af328de9e6e01d601ffdd7f4779f43cf5ed097e","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110291728471691264","occurred_at":"2026-10-14T08:48:48Z","signature_id":"b11574c6b2c9e56fb0df8243603cc190494eabc1514bd622eab98ffa66345c88","pr_number":7500,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":2.142857142857143,"anchor_occurrences":3,"after_last_push_count":1,"trend_counts":[1,2,1,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-10-05","2026-10-12"],"prior_runs_affected":16,"prior_last_seen_at":"2026-10-15T01:36:32Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T02:26:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j2839296-mgmt-2\", component=\"monitoring-infrastructure\", region=\"centralus\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T19:52:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j2981888-mgmt-2\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-14T16:54:29Z\n  Ended:       2026-10-14T17:00:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j8174976-mgmt-1\", component=\"monitoring-infrastructure\", region=\"centralus\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110545325142839296","occurred_at":"2026-10-15T01:36:32Z","signature_id":"050f4c440f64150e7970ac13f573a27b5f50a5f05fdeb18033f081e39ff717b6","pr_number":7133,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6946/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110441834562981888","occurred_at":"2026-10-14T18:45:15Z","signature_id":"4f5e43f8961ce1450724da4818b03afc4ea4a1e65b095d58ea6cf9018f4c082d","pr_number":6946,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110398398338174976","occurred_at":"2026-10-14T15:52:39Z","signature_id":"b44e1602655f10f93a02f5aad2f805d9843cc1136bdf9380aaa0cc48e7872d35","pr_number":7509,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9205c69c72a57356c2890f3b58a265a6f5459948112af87f37384e641208f260","failure_pattern":"ERROR CODE: 429; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: 429","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.4285714285714286,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[1,1,1,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-12"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-15T02:28:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:108]: failed to get service managed identity\nUnexpected error:\n    \u003c*exported.ResponseError | 0xf085da10420\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because tenant XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n            correlation ID: 34115d09-ce39-4c55-8529-6ab72e73b469\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:108]: failed to get service managed identity\nUnexpected error:\n    \u003c*exported.ResponseError | 0xf085da10420\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because tenant XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n            correlation ID: 34115d09-ce39-4c55-8529-6ab72e73b469\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:119]: failed to get cluster-api-azure managed identity\nUnexpected error:\n    \u003c*exported.ResponseError | 0x18d22f431050\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n            correlation ID: d277ac90-384a-44a9-9799-7698621627f9\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:119]: failed to get cluster-api-azure managed identity\nUnexpected error:\n    \u003c*exported.ResponseError | 0x18d22f431050\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n            correlation ID: d277ac90-384a-44a9-9799-7698621627f9\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558411769778176","occurred_at":"2026-10-15T02:28:32Z","signature_id":"48846a4a9ee7ad69b6a19d79cdae37682330e1e94cef9f146c300baeb116f561","pr_number":7342,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110168597706838016","occurred_at":"2026-10-14T00:39:33Z","signature_id":"e91976aae7db2919970076bbc44f0d95243241475acf02a79c2cb09c59930f22","pr_number":7488,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.4285714285714286,"anchor_occurrences":2,"after_last_push_count":9,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":17,"prior_last_seen_at":"2026-10-14T19:09:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0x15cfa7358820\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-kkjsbcrkrfqs\" to finish creating: GET https://rp.j3670144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d48f5df-4419-42d0-908d-f55c9e58b43d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d48f5df-4419-42d0-908d-f55c9e58b43d\",\n      \"name\": \"3d48f5df-4419-42d0-908d-f55c9e58b43d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T20:00:20.459546459Z\",\n      \"endTime\": \"2026-10-14T20:29:21.27372498Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.37 is Partial (want Completed), started 24m22s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3670144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d48f5df-4419-42d0-908d-f55c9e58b43d\n            correlation ID: 7a9ff375-e54e-4cef-a85f-da37d27bb541\n            client request ID: 20f4c917-baec-45d6-a1e1-7f0ef2326c1c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0x15cfa7358820\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-kkjsbcrkrfqs\" to finish creating: GET https://rp.j3670144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d48f5df-4419-42d0-908d-f55c9e58b43d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d48f5df-4419-42d0-908d-f55c9e58b43d\",\n      \"name\": \"3d48f5df-4419-42d0-908d-f55c9e58b43d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T20:00:20.459546459Z\",\n      \"endTime\": \"2026-10-14T20:29:21.27372498Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.37 is Partial (want Completed), started 24m22s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3670144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d48f5df-4419-42d0-908d-f55c9e58b43d\n            correlation ID: 7a9ff375-e54e-4cef-a85f-da37d27bb541\n            client request ID: 20f4c917-baec-45d6-a1e1-7f0ef2326c1c\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-8fzqsptw7bpr/cluster-candidate-5-0-g7bpwx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x4abdbffea60\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-g7bpwx: failed waiting for cluster=\"cluster-candidate-5-0-g7bpwx\" in resourcegroup=\"rg-candidate-5-0-8fzqsptw7bpr\" to finish creating: GET https://rp.j3579520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b237387d-d51f-41fb-8762-715d9d2b7454\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b237387d-d51f-41fb-8762-715d9d2b7454\",\n      \"name\": \"b237387d-d51f-41fb-8762-715d9d2b7454\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T07:05:39.437908469Z\",\n      \"endTime\": \"2026-10-14T07:34:48.061766511Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.6 is Partial (want Completed), started 12m48s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3579520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b237387d-d51f-41fb-8762-715d9d2b7454\n            correlation ID: 92af7c84-e19d-482a-a1fe-a87a2b603856\n            client request ID: 2dc5545b-5fc8-4471-80a2-89372581489c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-8fzqsptw7bpr/cluster-candidate-5-0-g7bpwx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x4abdbffea60\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-g7bpwx: failed waiting for cluster=\"cluster-candidate-5-0-g7bpwx\" in resourcegroup=\"rg-candidate-5-0-8fzqsptw7bpr\" to finish creating: GET https://rp.j3579520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b237387d-d51f-41fb-8762-715d9d2b7454\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b237387d-d51f-41fb-8762-715d9d2b7454\",\n      \"name\": \"b237387d-d51f-41fb-8762-715d9d2b7454\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T07:05:39.437908469Z\",\n      \"endTime\": \"2026-10-14T07:34:48.061766511Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.6 is Partial (want Completed), started 12m48s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3579520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b237387d-d51f-41fb-8762-715d9d2b7454\n            correlation ID: 92af7c84-e19d-482a-a1fe-a87a2b603856\n            client request ID: 2dc5545b-5fc8-4471-80a2-89372581489c\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110447856593670144","occurred_at":"2026-10-14T19:09:11Z","signature_id":"1d72d1eb80a478cce43c5b77b2461ed1a426d75211fc2d70cddaa21ad3a76436","pr_number":7529,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110251822873579520","occurred_at":"2026-10-14T06:10:15Z","signature_id":"afb8138a4654c4414a50bcc92d291c8bdf260bd4de2e39b80a917c8525fe79d5","pr_number":7504,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.4285714285714286,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["PROD","STG"],"trend_counts":[3,2,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-14T08:43:30Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-7tqpsp\" for cluster \"cluster-hs-pre-7tqpsp\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9fbbba82620\u003e: \n    failed to create NodePool np-7tqpsp: failed waiting for nodepool=\"np-7tqpsp\" for cluster \"cluster-hs-pre-7tqpsp\" in resourcegroup=\"rg-hs-pre--jlgwrtjbwxnf\" to finish creating: GET https://rp.j4313856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bae4fd0e-d65b-410a-a9d7-cb38c5e195e6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bae4fd0e-d65b-410a-a9d7-cb38c5e195e6\",\n      \"name\": \"bae4fd0e-d65b-410a-a9d7-cb38c5e195e6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T09:53:58.678670385Z\",\n      \"endTime\": \"2026-10-14T10:23:01.968138297Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a0b7j6n3g2s4x1b-np-7tqpsp-rqshb-xb9b7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--jlgwrtjbwxnf-7tqpsp--managed/providers/Microsoft.Compute/virtualMachines/a0b7j6n3g2s4x1b-np-7tqpsp-rqshb-xb9b7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a0b7j6n3g2s4x1b-np-7tqpsp-rqshb-xb9b7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--jlgwrtjbwxnf-7tqpsp--managed/providers/Microsoft.Compute/virtualMachines/a0b7j6n3g2s4x1b-np-7tqpsp-rqshb-xb9b7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4313856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bae4fd0e-d65b-410a-a9d7-cb38c5e195e6\n            correlation ID: fb25957f-8e08-4f05-8265-7dd58e255cf0\n            client request ID: 4be80cba-579f-4cb7-8b46-0f96bfb70faa\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-7tqpsp\" for cluster \"cluster-hs-pre-7tqpsp\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9fbbba82620\u003e: \n    failed to create NodePool np-7tqpsp: failed waiting for nodepool=\"np-7tqpsp\" for cluster \"cluster-hs-pre-7tqpsp\" in resourcegroup=\"rg-hs-pre--jlgwrtjbwxnf\" to finish creating: GET https://rp.j4313856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bae4fd0e-d65b-410a-a9d7-cb38c5e195e6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bae4fd0e-d65b-410a-a9d7-cb38c5e195e6\",\n      \"name\": \"bae4fd0e-d65b-410a-a9d7-cb38c5e195e6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T09:53:58.678670385Z\",\n      \"endTime\": \"2026-10-14T10:23:01.968138297Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a0b7j6n3g2s4x1b-np-7tqpsp-rqshb-xb9b7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--jlgwrtjbwxnf-7tqpsp--managed/providers/Microsoft.Compute/virtualMachines/a0b7j6n3g2s4x1b-np-7tqpsp-rqshb-xb9b7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a0b7j6n3g2s4x1b-np-7tqpsp-rqshb-xb9b7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--jlgwrtjbwxnf-7tqpsp--managed/providers/Microsoft.Compute/virtualMachines/a0b7j6n3g2s4x1b-np-7tqpsp-rqshb-xb9b7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4313856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bae4fd0e-d65b-410a-a9d7-cb38c5e195e6\n            correlation ID: fb25957f-8e08-4f05-8265-7dd58e255cf0\n            client request ID: 4be80cba-579f-4cb7-8b46-0f96bfb70faa\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-cxf7bl\" for cluster \"cluster-hs-pre-cxf7bl\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x7e11cac4ba0\u003e: \n    failed to create NodePool np-cxf7bl: failed waiting for nodepool=\"np-cxf7bl\" for cluster \"cluster-hs-pre-cxf7bl\" in resourcegroup=\"rg-hs-pre--zccxz29md4tk\" to finish creating: GET https://rp.j1740672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a6b9db2-bba7-47f4-b565-c775480b382a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a6b9db2-bba7-47f4-b565-c775480b382a\",\n      \"name\": \"2a6b9db2-bba7-47f4-b565-c775480b382a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T08:08:22.741490309Z\",\n      \"endTime\": \"2026-10-14T08:37:29.934217142Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d9f9j4p4h9j5g5y-np-cxf7bl-d29hq-jxkj6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--zccxz29md4tk-cxf7bl--managed/providers/Microsoft.Compute/virtualMachines/d9f9j4p4h9j5g5y-np-cxf7bl-d29hq-jxkj6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d9f9j4p4h9j5g5y-np-cxf7bl-d29hq-jxkj6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--zccxz29md4tk-cxf7bl--managed/providers/Microsoft.Compute/virtualMachines/d9f9j4p4h9j5g5y-np-cxf7bl-d29hq-jxkj6 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1740672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a6b9db2-bba7-47f4-b565-c775480b382a\n            correlation ID: ca4a0e92-cce5-474f-a388-05ef9eae8408\n            client request ID: 843e4749-4432-4dd5-a384-22418cffc290\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-cxf7bl\" for cluster \"cluster-hs-pre-cxf7bl\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x7e11cac4ba0\u003e: \n    failed to create NodePool np-cxf7bl: failed waiting for nodepool=\"np-cxf7bl\" for cluster \"cluster-hs-pre-cxf7bl\" in resourcegroup=\"rg-hs-pre--zccxz29md4tk\" to finish creating: GET https://rp.j1740672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a6b9db2-bba7-47f4-b565-c775480b382a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a6b9db2-bba7-47f4-b565-c775480b382a\",\n      \"name\": \"2a6b9db2-bba7-47f4-b565-c775480b382a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T08:08:22.741490309Z\",\n      \"endTime\": \"2026-10-14T08:37:29.934217142Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d9f9j4p4h9j5g5y-np-cxf7bl-d29hq-jxkj6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--zccxz29md4tk-cxf7bl--managed/providers/Microsoft.Compute/virtualMachines/d9f9j4p4h9j5g5y-np-cxf7bl-d29hq-jxkj6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d9f9j4p4h9j5g5y-np-cxf7bl-d29hq-jxkj6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--zccxz29md4tk-cxf7bl--managed/providers/Microsoft.Compute/virtualMachines/d9f9j4p4h9j5g5y-np-cxf7bl-d29hq-jxkj6 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1740672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a6b9db2-bba7-47f4-b565-c775480b382a\n            correlation ID: ca4a0e92-cce5-474f-a388-05ef9eae8408\n            client request ID: 843e4749-4432-4dd5-a384-22418cffc290\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7515/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110290387544313856","occurred_at":"2026-10-14T08:43:30Z","signature_id":"50e6f316011a8296fa2af306e18ddc7dbd6a5f66fbf21de99e30ffc49bba1879","pr_number":7515,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110267140211740672","occurred_at":"2026-10-14T07:11:07Z","signature_id":"6b14ed8a4511df290f8bf9163c6eafbbcf710dd60c50bd84885e6283a1516ec7","pr_number":7451,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9220c54af3cdcf82280969ee7ca39e91829e564fcd5ac622742ec149542a29fd","failure_pattern":"ERROR CODE: DeploymentFailed; detail code HttpRequestException; detail message Error while processing network security perimeter association: An error occurred while sending the request.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=2)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.4285714285714286,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-10-12"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-14T18:34:47Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=2)","occurrences":2}],"full_error_samples":["time=2026-10-14T19:14:18.297Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 description=\"Step nsp\\n  Kind: ARM\\n  Template: templates/mgmt-nsp.bicep\\n  Parameters: configurations/mgmt-nsp.tmpl.bicepparam\"\ntime=2026-10-14T19:14:18.788Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2\ntime=2026-10-14T19:14:19.241Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 deployment=cf8c9a34812ddee83e87f5e1fcb73992e7f0046ac9c8abee6011c27a19ac7c31 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9264896-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcf8c9a34812ddee83e87f5e1fcb73992e7f0046ac9c8abee6011c27a19ac7c31\ntime=2026-10-14T19:14:49.370Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9264896-mgmt-2/providers/Microsoft.Resources/deployments/cf8c9a34812ddee83e87f5e1fcb73992e7f0046ac9c8abee6011c27a19ac7c31/operationStatuses/08584096016264886235\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9264896-mgmt-2/providers/Microsoft.Resources/deployments/profile-xpjnxfpmcvora\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9264896-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j9264896-mgmt-2/resourceAssociations/nsp-j9264896-mgmt-2-ra-a9c395a5-2fe0-5927-97a4-6856e96269ae\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"HttpRequestException\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9264896-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j9264896-mgmt-2/resourceAssociations/nsp-j9264896-mgmt-2-ra-a9c395a5-2fe0-5927-97a4-6856e96269ae, Error message: An error occurred while sending the request.\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-10-14T15:57:00.478Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 description=\"Step nsp\\n  Kind: ARM\\n  Template: templates/mgmt-nsp.bicep\\n  Parameters: configurations/mgmt-nsp.tmpl.bicepparam\"\ntime=2026-10-14T15:57:01.032Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2\ntime=2026-10-14T15:57:01.677Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 deployment=9e882a024304e709a0349553a748b62ffe4602dc3aa507f98e24a10ee21a2af8 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7217792-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9e882a024304e709a0349553a748b62ffe4602dc3aa507f98e24a10ee21a2af8\ntime=2026-10-14T15:57:32.150Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7217792-mgmt-2/providers/Microsoft.Resources/deployments/9e882a024304e709a0349553a748b62ffe4602dc3aa507f98e24a10ee21a2af8/operationStatuses/08584096134641768649\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7217792-mgmt-2/providers/Microsoft.Resources/deployments/profile-7bczjqgoi25xw\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7217792-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j7217792-mgmt-2/resourceAssociations/nsp-j7217792-mgmt-2-ra-e5d09873-b78f-5e95-bab4-2921347dc438\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"HttpRequestException\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7217792-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j7217792-mgmt-2/resourceAssociations/nsp-j7217792-mgmt-2-ra-e5d09873-b78f-5e95-bab4-2921347dc438, Error message: An error occurred while sending the request.\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110439188909264896","occurred_at":"2026-10-14T18:34:47Z","signature_id":"d465f96a6ebaf98458f369563694ed7b4f0235597cd9af66af75425f966c9780","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110390267277217792","occurred_at":"2026-10-14T15:20:23Z","signature_id":"d465f96a6ebaf98458f369563694ed7b4f0235597cd9af66af75425f966c9780","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1a53b0c30b515bff265c32e9b65ff52445c52775e0172c5df823b4597bd77e2b","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.4285714285714286,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[1,2,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-14T19:01:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:320]: Unexpected error:\n    \u003c*errors.joinError | 0x1e1ed182150\u003e: \n    resource group \"rg-candidate-4-21-xnx2svpmndkd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-bqkz5m\" in resourcegroup=\"rg-candidate-4-21-xnx2svpmndkd\" to finish deleting: GET https://rp.j6397952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aefc9a01-d084-42f4-9d9d-673831774ba7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aefc9a01-d084-42f4-9d9d-673831774ba7\",\n      \"name\": \"aefc9a01-d084-42f4-9d9d-673831774ba7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T20:07:21.117969363Z\",\n      \"endTime\": \"2026-10-14T20:31:23.904548294Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6397952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aefc9a01-d084-42f4-9d9d-673831774ba7\n            correlation ID: e7f218c6-4d7a-4ce1-8044-c7ec05ec7356\n            client request ID: e554513d-44da-4c58-8dae-c72863b0374d\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:320]: Unexpected error:\n    \u003c*errors.joinError | 0x1e1ed182150\u003e: \n    resource group \"rg-candidate-4-21-xnx2svpmndkd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-bqkz5m\" in resourcegroup=\"rg-candidate-4-21-xnx2svpmndkd\" to finish deleting: GET https://rp.j6397952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aefc9a01-d084-42f4-9d9d-673831774ba7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aefc9a01-d084-42f4-9d9d-673831774ba7\",\n      \"name\": \"aefc9a01-d084-42f4-9d9d-673831774ba7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T20:07:21.117969363Z\",\n      \"endTime\": \"2026-10-14T20:31:23.904548294Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6397952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aefc9a01-d084-42f4-9d9d-673831774ba7\n            correlation ID: e7f218c6-4d7a-4ce1-8044-c7ec05ec7356\n            client request ID: e554513d-44da-4c58-8dae-c72863b0374d\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x50b30376378\u003e: \n    resource group \"rg-candidate-4-23-zgqgjh9b82nm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-v7mn4s\" in resourcegroup=\"rg-candidate-4-23-zgqgjh9b82nm\" to finish deleting: GET https://rp.j9234816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90b34b3b-49e0-48b7-b175-fed3e7efcffe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90b34b3b-49e0-48b7-b175-fed3e7efcffe\",\n      \"name\": \"90b34b3b-49e0-48b7-b175-fed3e7efcffe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T15:55:23.629401759Z\",\n      \"endTime\": \"2026-10-14T16:19:25.512560555Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9234816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90b34b3b-49e0-48b7-b175-fed3e7efcffe\n            correlation ID: 18b3d50f-1bf0-4dc5-95df-d87efa3be15d\n            client request ID: eef778ec-0385-499a-a74c-a4ea9c9eb634\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x50b30376378\u003e: \n    resource group \"rg-candidate-4-23-zgqgjh9b82nm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-v7mn4s\" in resourcegroup=\"rg-candidate-4-23-zgqgjh9b82nm\" to finish deleting: GET https://rp.j9234816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90b34b3b-49e0-48b7-b175-fed3e7efcffe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90b34b3b-49e0-48b7-b175-fed3e7efcffe\",\n      \"name\": \"90b34b3b-49e0-48b7-b175-fed3e7efcffe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T15:55:23.629401759Z\",\n      \"endTime\": \"2026-10-14T16:19:25.512560555Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9234816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90b34b3b-49e0-48b7-b175-fed3e7efcffe\n            correlation ID: 18b3d50f-1bf0-4dc5-95df-d87efa3be15d\n            client request ID: eef778ec-0385-499a-a74c-a4ea9c9eb634\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110445968326397952","occurred_at":"2026-10-14T19:01:26Z","signature_id":"9f4ffc5eb425744bf1f26b0e1f04b2d4faa6a9bb922b93d34f3023ef78ad7eed","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7428/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110376503379234816","occurred_at":"2026-10-14T14:25:41Z","signature_id":"f07da3d933a833699409e75c9aa7df75da760b199cbfea3580c7225c9d2f11ef","pr_number":7428,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"de36b52ee87d849c34b3bf8b0df9a50c73bc24fcb413c756452123c1118a8064","failure_pattern":"There is no failure as the matcher passed to Eventually succeeded on its most recent iteration","search_query":"There is no failure as the matcher passed to Eventually succeeded on its most recent iteration","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.4285714285714286,"anchor_occurrences":2,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-14T23:59:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:394]: Timed out after 90.003s.\ndelegation and authoritative NS answers should remain consistent for at least 30 seconds before issuance\nThere is no failure as the matcher passed to Eventually succeeded on its most recent iteration\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:394]: Timed out after 90.003s.\ndelegation and authoritative NS answers should remain consistent for at least 30 seconds before issuance\nThere is no failure as the matcher passed to Eventually succeeded on its most recent iteration","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:394]: Timed out after 90.000s.\ndelegation and authoritative NS answers should remain consistent for at least 30 seconds before issuance\nThere is no failure as the matcher passed to Eventually succeeded on its most recent iteration\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:394]: Timed out after 90.000s.\ndelegation and authoritative NS answers should remain consistent for at least 30 seconds before issuance\nThere is no failure as the matcher passed to Eventually succeeded on its most recent iteration"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7544/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110520946510008320","occurred_at":"2026-10-14T23:59:39Z","signature_id":"19450299063bae600bd538ea8d3934944e65d651ee47d4e5a838f879f17c9128","pr_number":7544,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110389077940375552","occurred_at":"2026-10-14T15:15:39Z","signature_id":"7eeadc6a06fe33387af72d93bce8e84c520e130092931b2582b514a2cfa0eeda","pr_number":7492,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"bff205290781819ab19bf99f50804120a81da4620fe1fb58b29673eae7587c8f","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T04:13:54.2670610Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\", ActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", SubActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a \u003e GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\"}, IsPermanent:false})","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.4285714285714286,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,1,1,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-15T02:28:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c33df13c000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T04:13:54.2670610Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\", ActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", SubActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a \u003e GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T04:13:54.2670610Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\\\", ActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", SubActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a \u003e GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x3c33df13c540\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T04:13:54.2670610Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\\\", ActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", SubActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a \u003e GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x3c33df6e8000\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-15T04:13:54.2670610Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\",\n                        ActivityId: \"11bf0618-c400-4197-b3c0-0589df8f658a\",\n                        SubActivityId: \"11bf0618-c400-4197-b3c0-0589df8f658a\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"11bf0618-c400-4197-b3c0-0589df8f658a\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a \u003e GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c33df13c000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T04:13:54.2670610Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\", ActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", SubActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a \u003e GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T04:13:54.2670610Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\\\", ActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", SubActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a \u003e GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x3c33df13c540\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T04:13:54.2670610Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\\\", ActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", SubActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a \u003e GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x3c33df6e8000\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-15T04:13:54.2670610Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\",\n                        ActivityId: \"11bf0618-c400-4197-b3c0-0589df8f658a\",\n                        SubActivityId: \"11bf0618-c400-4197-b3c0-0589df8f658a\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"11bf0618-c400-4197-b3c0-0589df8f658a\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a \u003e GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tgr8grhvqs5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a8ac5b76060\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tgr8grhvqs5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T08:31:36.3290647Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\", ActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", SubActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 \u003e GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tgr8grhvqs5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T08:31:36.3290647Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\\\", ActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", SubActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 \u003e GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x2a8ac5f001c0\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T08:31:36.3290647Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\\\", ActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", SubActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 \u003e GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x2a8ac67f4100\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-14T08:31:36.3290647Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\",\n                        ActivityId: \"aec379bd-7447-4ac4-a50b-420184492383\",\n                        SubActivityId: \"aec379bd-7447-4ac4-a50b-420184492383\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"aec379bd-7447-4ac4-a50b-420184492383\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 \u003e GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tgr8grhvqs5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a8ac5b76060\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tgr8grhvqs5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T08:31:36.3290647Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\", ActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", SubActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 \u003e GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tgr8grhvqs5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T08:31:36.3290647Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\\\", ActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", SubActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 \u003e GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x2a8ac5f001c0\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T08:31:36.3290647Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\\\", ActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", SubActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 \u003e GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x2a8ac67f4100\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-14T08:31:36.3290647Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\",\n                        ActivityId: \"aec379bd-7447-4ac4-a50b-420184492383\",\n                        SubActivityId: \"aec379bd-7447-4ac4-a50b-420184492383\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"aec379bd-7447-4ac4-a50b-420184492383\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 \u003e GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558411769778176","occurred_at":"2026-10-15T02:28:32Z","signature_id":"770ac06f7acb59154eca862595eb4bf4400b3c61cdce89dc281592eb5542d356","pr_number":7342,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110265029910597632","occurred_at":"2026-10-14T07:02:43Z","signature_id":"c88d750e35c7c0d4748d9eea15750ad38b0e7332be0d2e329a4d176a4e356283","pr_number":7516,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.4285714285714286,"anchor_occurrences":2,"after_last_push_count":7,"trend_counts":[3,2,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":42,"prior_last_seen_at":"2026-10-14T22:15:16Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":2}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-14T23:34:33Z\n  Ended:       2026-10-14T23:51:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0214528-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0214528/providers/microsoft.monitor/accounts/hcps-j0214528\", namespace=\"ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-r2hbm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-14T23:35:30Z\n  Ended:       2026-10-14T23:52:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0214528-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0214528/providers/microsoft.monitor/accounts/hcps-j0214528\", namespace=\"ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-r2hbm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-14T10:09:45Z\n  Ended:       2026-10-14T10:36:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j1691264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.235:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1691264/providers/microsoft.monitor/accounts/hcps-j1691264\", namespace=\"ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-9qldk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-14T10:09:45Z\n  Ended:       2026-10-14T10:36:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j1691264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.235:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1691264/providers/microsoft.monitor/accounts/hcps-j1691264\", namespace=\"ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-9qldk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110494690280214528","occurred_at":"2026-10-14T22:15:16Z","signature_id":"30bbfbad94a3e968ca9a9fdf6b8a8083382f6eb5df9064ba2d12e6b538f69cff","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110291728471691264","occurred_at":"2026-10-14T08:48:48Z","signature_id":"02ab642b00e1ec0d87eac69fdef749538d89f3c679b714b14c0e98e0b084fe7c","pr_number":7500,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"abb1429e70773e39316bd0948d4a9486cc702f42873918d630fe98006b70e184","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=1)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.4285714285714286,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,1,1,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-10-05","2026-10-12"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-15T01:36:32Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=1)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-15T02:14:27.753Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 description=\"Step mitigations\\n Kind: Helm\\n\"\ntime=2026-10-15T02:14:28.197Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 values=\"map[azureCnsCpuLimits:map[enabled:true] jobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157]\"\ntime=2026-10-15T02:14:28.197Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-15T02:14:28.543Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 namespace=aro-hcp-mitigations\ntime=2026-10-15T02:14:28.648Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 dryRun=false\ntime=2026-10-15T02:14:28.648Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-15T02:14:28.690Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-15T02:20:27.812Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-15T02:20:27.812Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"","time=2026-10-14T18:50:15.283Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 description=\"Step mitigations\\n Kind: Helm\\n\"\ntime=2026-10-14T18:50:15.838Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 values=\"map[azureCnsCpuLimits:map[enabled:true] jobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157]\"\ntime=2026-10-14T18:50:15.838Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1\ntime=2026-10-14T18:50:16.026Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 namespace=aro-hcp-mitigations\ntime=2026-10-14T18:50:16.350Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 dryRun=false\ntime=2026-10-14T18:50:16.350Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1\ntime=2026-10-14T18:50:16.477Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1\ntime=2026-10-14T18:56:25.364Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 err=\"failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-14T18:56:25.364Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110545325142839296","occurred_at":"2026-10-15T01:36:32Z","signature_id":"36808247d658b9b22b59f418949b27fa87b1655933f39638ef6bec47bfa0327e","pr_number":7133,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110431880728809472","occurred_at":"2026-10-14T18:05:44Z","signature_id":"3eea31dd2ba80ae3c26eda9638a5c4961c637f75b7b51fe4771e43332561b111","pr_number":7133,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-2n7cffz9j9vb\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.4285714285714286,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":9,"prior_last_seen_at":"2026-10-14T18:22:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x1399efb46de0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-2n7cffz9j9vb\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-2n7cffz9j9vb\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x1399ef809d30\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x1399efb46de0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-2n7cffz9j9vb\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-2n7cffz9j9vb\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x1399ef809d30\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x14825bb062d0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-gvb49lpzrwdc\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-gvb49lpzrwdc, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-gvb49lpzrwdc\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-gvb49lpzrwdc, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x14825b6451e0\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-gvb49lpzrwdc\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x14825bb062d0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-gvb49lpzrwdc\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-gvb49lpzrwdc, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-gvb49lpzrwdc\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-gvb49lpzrwdc, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x14825b6451e0\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-gvb49lpzrwdc\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110436010226618368","occurred_at":"2026-10-14T18:22:06Z","signature_id":"f01d09e5b26ed293c1f2fb9f79dc527c3d66d0ac5cf3f21d86318c9622720514","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"ac0f754d7b800e55b8405f91d30db0dad28eccf8fd68556cf99e019a8b65a48f","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9310a4a1ba70b92dcd897ea7ccef6ff3d68afd4098649e3461e4758a9ed7df2f","failure_pattern":"VerifyIngressControllerScope(External) timed out after 5m0","search_query":"VerifyIngressControllerScope(External) timed out after 5m0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T05:04:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:199]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2c6abc8c6320\u003e: \n    VerifyIngressControllerScope(External) timed out after 5m0.002s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyIngressControllerScope(External) timed out after 5m0.002s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2c6abc8c62a0\u003e{\n            msg: \"failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x2c6abc8c6260\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:199]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2c6abc8c6320\u003e: \n    VerifyIngressControllerScope(External) timed out after 5m0.002s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyIngressControllerScope(External) timed out after 5m0.002s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2c6abc8c62a0\u003e{\n            msg: \"failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x2c6abc8c6260\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:199]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1ba452ab87c0\u003e: \n    VerifyIngressControllerScope(External) timed out after 5m0.006s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyIngressControllerScope(External) timed out after 5m0.006s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1ba452ab87a0\u003e{\n            msg: \"failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x1ba452ab8760\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:199]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1ba452ab87c0\u003e: \n    VerifyIngressControllerScope(External) timed out after 5m0.006s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyIngressControllerScope(External) timed out after 5m0.006s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1ba452ab87a0\u003e{\n            msg: \"failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x1ba452ab8760\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110235250528358400","occurred_at":"2026-10-14T05:04:24Z","signature_id":"7d84995ad741c9681c8c3c699502f42032152628033a5acce166137600455589","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110235250528358400","occurred_at":"2026-10-14T05:04:24Z","signature_id":"c488b2162ac9532ad63b94efbbfbaac3551ef53bea0886a3d47dcc0d5f92dd8e","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f3a7b19a161f6d450e990aabc7d01a1298ee43d7b8bea6fe710061141d7aca91","failure_pattern":"All ClusterOperators should report Available=True, but these are not available: [console (Available=False)]","search_query":"All ClusterOperators should report Available=True, but these are not available: [console (Available=False)]","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T16:49:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:378]: Timed out after 639.547s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:377 with:\nAll ClusterOperators should report Available=True, but these are not available: [console (Available=False)]\nExpected\n    \u003c[]string | len:1, cap:1\u003e: [\n        \"console (Available=False)\",\n    ]\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:378]: Timed out after 639.547s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:377 with:\nAll ClusterOperators should report Available=True, but these are not available: [console (Available=False)]\nExpected\n    \u003c[]string | len:1, cap:1\u003e: [\n        \"console (Available=False)\",\n    ]\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110412598238253056","occurred_at":"2026-10-14T16:49:06Z","signature_id":"e27d6f0c138773167813ece164429b7c33de4079b9a5f054e3aa5a8fdc5f3e0f","pr_number":7529,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"fa65c8375667d2f86c4c029a1dd32730b8452c4c21edd3768c8aaedc846a9603","failure_pattern":"ERROR CODE: Canceled; detail message This operation was superseded by another; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Canceled","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-15T03:25:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-mpppvh with version 4.21.37\nUnexpected error:\n    \u003c*fmt.wrapError | 0x279027f833e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-mpppvh: failed waiting for cluster=\"np-version-upgrade-cluster-mpppvh\" in resourcegroup=\"rg-np-version-upgrade-mpppvh-6ckrm5njjkhq\" to finish creating: GET https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7ef9bc0-8329-437a-9805-6b4b65fd8b47\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: Canceled\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7ef9bc0-8329-437a-9805-6b4b65fd8b47\",\n      \"name\": \"a7ef9bc0-8329-437a-9805-6b4b65fd8b47\",\n      \"status\": \"Canceled\",\n      \"startTime\": \"2026-10-15T04:31:31.170202136Z\",\n      \"endTime\": \"2026-10-15T04:31:34.719444109Z\",\n      \"error\": {\n        \"code\": \"Canceled\",\n        \"message\": \"This operation was superseded by another\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7ef9bc0-8329-437a-9805-6b4b65fd8b47\n            correlation ID: bbacf445-8423-4177-9af5-a54728857aa3\n            client request ID: 33a84f8d-a321-43e3-a649-cc65e487871e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-mpppvh with version 4.21.37\nUnexpected error:\n    \u003c*fmt.wrapError | 0x279027f833e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-mpppvh: failed waiting for cluster=\"np-version-upgrade-cluster-mpppvh\" in resourcegroup=\"rg-np-version-upgrade-mpppvh-6ckrm5njjkhq\" to finish creating: GET https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7ef9bc0-8329-437a-9805-6b4b65fd8b47\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: Canceled\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7ef9bc0-8329-437a-9805-6b4b65fd8b47\",\n      \"name\": \"a7ef9bc0-8329-437a-9805-6b4b65fd8b47\",\n      \"status\": \"Canceled\",\n      \"startTime\": \"2026-10-15T04:31:31.170202136Z\",\n      \"endTime\": \"2026-10-15T04:31:34.719444109Z\",\n      \"error\": {\n        \"code\": \"Canceled\",\n        \"message\": \"This operation was superseded by another\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7ef9bc0-8329-437a-9805-6b4b65fd8b47\n            correlation ID: bbacf445-8423-4177-9af5-a54728857aa3\n            client request ID: 33a84f8d-a321-43e3-a649-cc65e487871e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7401/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110572658033692672","occurred_at":"2026-10-15T03:25:08Z","signature_id":"3463f8e6114f7652dff0d4ce4c5cc8eb355bff4de07583420b62c0e190564e47","pr_number":7401,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1019af4c9c47d1aa706702f78998f006a0e49b021ad920d44d356df2637fa18e","failure_pattern":"ERROR CODE: Conflict; detail message Cannot update resource while resource is \"accepted\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-15T03:25:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-mpppvh with version 4.21.37\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1bc8b1824920\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-mpppvh: failed starting cluster creation \"np-version-upgrade-cluster-mpppvh\" in resourcegroup=\"rg-np-version-upgrade-mpppvh-6ckrm5njjkhq\": PUT https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mpppvh-6ckrm5njjkhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mpppvh\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot update resource while resource is \\\"accepted\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mpppvh-6ckrm5njjkhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mpppvh\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mpppvh-6ckrm5njjkhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mpppvh\n            correlation ID: 5672e540-0082-400e-8d95-fd1926fa87e0\n            client request ID: 32f4cab2-abf5-42a2-aa1e-6a050c192397\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-mpppvh with version 4.21.37\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1bc8b1824920\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-mpppvh: failed starting cluster creation \"np-version-upgrade-cluster-mpppvh\" in resourcegroup=\"rg-np-version-upgrade-mpppvh-6ckrm5njjkhq\": PUT https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mpppvh-6ckrm5njjkhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mpppvh\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot update resource while resource is \\\"accepted\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mpppvh-6ckrm5njjkhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mpppvh\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mpppvh-6ckrm5njjkhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mpppvh\n            correlation ID: 5672e540-0082-400e-8d95-fd1926fa87e0\n            client request ID: 32f4cab2-abf5-42a2-aa1e-6a050c192397\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7401/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110572658033692672","occurred_at":"2026-10-15T03:25:08Z","signature_id":"3747b30e570e4b1c00673a1f97fdb92c7523478b496e1ded3b8cf9cde8d49d23","pr_number":7401,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2c0a88f2a0ef84e4c12e30b05c3779aee6f0cf66551b88fac3c1693eccdbecc5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-14T20:56:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d5895186200\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-knlckq8m42ph\" to finish creating: GET https://rp.j7016064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b31da953-f506-41e7-9fc1-8b942d4e8068\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b31da953-f506-41e7-9fc1-8b942d4e8068\",\n      \"name\": \"b31da953-f506-41e7-9fc1-8b942d4e8068\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T21:44:40.287298623Z\",\n      \"endTime\": \"2026-10-14T22:13:42.52978598Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-network-operator, catalog-operator, packageserver, openshift-route-controller-manager, cluster-policy-controller, cluster-storage-operator, control-plane-pki-operator, openshift-controller-manager, dns-operator, kube-controller-manager, cluster-node-tuning-operator, olm-operator, redhat-operators-catalog, certified-operators-catalog, csi-snapshot-controller-operator, machine-approver, azure-cloud-controller-manager, cluster-autoscaler, openshift-apiserver, hosted-cluster-config-operator, redhat-marketplace-catalog, community-operators-catalog, cluster-version-operator, ingress-operator, cluster-image-registry-operator, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7016064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b31da953-f506-41e7-9fc1-8b942d4e8068\n            correlation ID: 4b021430-3a3e-4288-80c7-fd56abc38462\n            client request ID: bf735dc6-c8ef-42d7-978e-a9ed4b21e87f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d5895186200\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-knlckq8m42ph\" to finish creating: GET https://rp.j7016064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b31da953-f506-41e7-9fc1-8b942d4e8068\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b31da953-f506-41e7-9fc1-8b942d4e8068\",\n      \"name\": \"b31da953-f506-41e7-9fc1-8b942d4e8068\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T21:44:40.287298623Z\",\n      \"endTime\": \"2026-10-14T22:13:42.52978598Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-network-operator, catalog-operator, packageserver, openshift-route-controller-manager, cluster-policy-controller, cluster-storage-operator, control-plane-pki-operator, openshift-controller-manager, dns-operator, kube-controller-manager, cluster-node-tuning-operator, olm-operator, redhat-operators-catalog, certified-operators-catalog, csi-snapshot-controller-operator, machine-approver, azure-cloud-controller-manager, cluster-autoscaler, openshift-apiserver, hosted-cluster-config-operator, redhat-marketplace-catalog, community-operators-catalog, cluster-version-operator, ingress-operator, cluster-image-registry-operator, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7016064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b31da953-f506-41e7-9fc1-8b942d4e8068\n            correlation ID: 4b021430-3a3e-4288-80c7-fd56abc38462\n            client request ID: bf735dc6-c8ef-42d7-978e-a9ed4b21e87f\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7527/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110474746717016064","occurred_at":"2026-10-14T20:56:04Z","signature_id":"48f5d6fe28a1e01e349727dd8c1e637671c49c3f0a083af8d28b1fa2eadaaa1d","pr_number":7527,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[1,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-12"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-14T12:43:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0x175f66e5a600\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-8brvsr9whg7b\" to finish creating: GET https://rp.j2105600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cba8c76e-06a0-4c2a-b3e0-d699a7abb497\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cba8c76e-06a0-4c2a-b3e0-d699a7abb497\",\n      \"name\": \"cba8c76e-06a0-4c2a-b3e0-d699a7abb497\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T13:47:00.778542361Z\",\n      \"endTime\": \"2026-10-14T14:16:06.716990034Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.37 is Partial (want Completed), started 17m32s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2105600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cba8c76e-06a0-4c2a-b3e0-d699a7abb497\n            correlation ID: 113b689f-35da-4e51-8207-8a80d51d19ea\n            client request ID: d9df4b89-ae88-4991-9a25-651fec75b816\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0x175f66e5a600\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-8brvsr9whg7b\" to finish creating: GET https://rp.j2105600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cba8c76e-06a0-4c2a-b3e0-d699a7abb497\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cba8c76e-06a0-4c2a-b3e0-d699a7abb497\",\n      \"name\": \"cba8c76e-06a0-4c2a-b3e0-d699a7abb497\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T13:47:00.778542361Z\",\n      \"endTime\": \"2026-10-14T14:16:06.716990034Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.37 is Partial (want Completed), started 17m32s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2105600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cba8c76e-06a0-4c2a-b3e0-d699a7abb497\n            correlation ID: 113b689f-35da-4e51-8207-8a80d51d19ea\n            client request ID: d9df4b89-ae88-4991-9a25-651fec75b816\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7522/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110350863582105600","occurred_at":"2026-10-14T12:43:47Z","signature_id":"fdd70b6d05559ad6a1e3f80813310696d29fbbfa70255c4f081213256f2c33ce","pr_number":7522,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4e831f602c4ea7f0cce0ce97a860291c59c5abae603b8e4d909a59a3f966062d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-\u003cid\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-\u003cid\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T08:48:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2e793540e048\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4b88pr2h6f62\" to finish creating: GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02548047-b3fd-49b8-8426-7df711bcbfbe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02548047-b3fd-49b8-8426-7df711bcbfbe\",\n      \"name\": \"02548047-b3fd-49b8-8426-7df711bcbfbe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T10:01:14.415831901Z\",\n      \"endTime\": \"2026-10-14T10:30:17.083278262Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-794ws-42z9w: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-42z9w to exist\\nMachine cilium-cluster-cilium-np-794ws-44qrb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-44qrb to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-794ws-42z9w: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-42z9w to exist\\nMachine cilium-cluster-cilium-np-794ws-44qrb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-44qrb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02548047-b3fd-49b8-8426-7df711bcbfbe\n            correlation ID: d40a80e8-ffc2-4aca-bd50-98d7806c38e8\n            client request ID: 84b1cf39-1ee3-4c6a-80ac-30f66febe153\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2e793540e048\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4b88pr2h6f62\" to finish creating: GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02548047-b3fd-49b8-8426-7df711bcbfbe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02548047-b3fd-49b8-8426-7df711bcbfbe\",\n      \"name\": \"02548047-b3fd-49b8-8426-7df711bcbfbe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T10:01:14.415831901Z\",\n      \"endTime\": \"2026-10-14T10:30:17.083278262Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-794ws-42z9w: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-42z9w to exist\\nMachine cilium-cluster-cilium-np-794ws-44qrb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-44qrb to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-794ws-42z9w: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-42z9w to exist\\nMachine cilium-cluster-cilium-np-794ws-44qrb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-44qrb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02548047-b3fd-49b8-8426-7df711bcbfbe\n            correlation ID: d40a80e8-ffc2-4aca-bd50-98d7806c38e8\n            client request ID: 84b1cf39-1ee3-4c6a-80ac-30f66febe153\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110291728471691264","occurred_at":"2026-10-14T08:48:48Z","signature_id":"d276781ea9d969dec523b2e2c4626568e236e9931f92b3fabae1fbe0cc6e76bf","pr_number":7500,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2a17ea6624b520ae513ca4725852ca83305c70a471a12e833ed2d7d18cfe8aaa","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-\u003cid\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-\u003cid\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T00:39:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x147375bd60d8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-l8lntszfbqhk\" to finish creating: GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a\",\n      \"name\": \"1bccabc3-2126-4728-9d49-4eea01285f0a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T01:45:21.225915122Z\",\n      \"endTime\": \"2026-10-14T02:14:22.961369975Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-llxzk-7ntxq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-7ntxq to exist\\nMachine cilium-cluster-cilium-np-llxzk-gsvtb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-llxzk-7ntxq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-7ntxq to exist\\nMachine cilium-cluster-cilium-np-llxzk-gsvtb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a\n            correlation ID: 854376b3-85ce-463a-aabe-5fc52fac910e\n            client request ID: d86cb5ff-fe28-499e-baef-054a0de770d9\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x147375bd60d8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-l8lntszfbqhk\" to finish creating: GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a\",\n      \"name\": \"1bccabc3-2126-4728-9d49-4eea01285f0a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T01:45:21.225915122Z\",\n      \"endTime\": \"2026-10-14T02:14:22.961369975Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-llxzk-7ntxq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-7ntxq to exist\\nMachine cilium-cluster-cilium-np-llxzk-gsvtb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-llxzk-7ntxq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-7ntxq to exist\\nMachine cilium-cluster-cilium-np-llxzk-gsvtb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a\n            correlation ID: 854376b3-85ce-463a-aabe-5fc52fac910e\n            client request ID: d86cb5ff-fe28-499e-baef-054a0de770d9\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110168566228586496","occurred_at":"2026-10-14T00:39:25Z","signature_id":"d1401fd1962c2b0a341d2d8996ab42716af9f4edd450ddb234863dd75ed0017e","pr_number":7248,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"9753620db9bca915ac646a67bd2e0c008b9f7be56f68db99d81d98334bd0e945","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-6rhdr-cwgtg to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-6rhdr-cwgtg to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T19:26:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x190aa1492400\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-qbnc288qgj6j\" to finish creating: GET https://rp.j3865088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199b8626-fe4e-4b24-9607-bad9bdfbcbe7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199b8626-fe4e-4b24-9607-bad9bdfbcbe7\",\n      \"name\": \"199b8626-fe4e-4b24-9607-bad9bdfbcbe7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T20:29:55.420741799Z\",\n      \"endTime\": \"2026-10-14T20:59:01.637394871Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-6rhdr-cwgtg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-qbnc288qgj6j--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-6rhdr-cwgtg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-6rhdr-cwgtg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-qbnc288qgj6j--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-6rhdr-cwgtg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3865088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199b8626-fe4e-4b24-9607-bad9bdfbcbe7\n            correlation ID: 1367eb49-5332-4cf5-9a4a-4b806be9f934\n            client request ID: 9a4866d1-15fd-4118-b349-e831018d7dac\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x190aa1492400\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-qbnc288qgj6j\" to finish creating: GET https://rp.j3865088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199b8626-fe4e-4b24-9607-bad9bdfbcbe7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199b8626-fe4e-4b24-9607-bad9bdfbcbe7\",\n      \"name\": \"199b8626-fe4e-4b24-9607-bad9bdfbcbe7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T20:29:55.420741799Z\",\n      \"endTime\": \"2026-10-14T20:59:01.637394871Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-6rhdr-cwgtg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-qbnc288qgj6j--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-6rhdr-cwgtg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-6rhdr-cwgtg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-qbnc288qgj6j--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-6rhdr-cwgtg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3865088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199b8626-fe4e-4b24-9607-bad9bdfbcbe7\n            correlation ID: 1367eb49-5332-4cf5-9a4a-4b806be9f934\n            client request ID: 9a4866d1-15fd-4118-b349-e831018d7dac\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452260763865088","occurred_at":"2026-10-14T19:26:43Z","signature_id":"be91c275aa99e6fe4f3186dff45127b70f53e8965e714209fe971eb77d5dcd19","pr_number":7519,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"08c1c1b279d699e6f9a43271a8c32e8779da05f6b9c675b49ef7772e07d5726d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-nzqkm-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-nzqkm-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T06:41:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a53ed53e100\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-b6z74shd5q4f\" to finish creating: GET https://rp.j0292224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e37b257b-a800-4d32-9117-6f5dadd22fa8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e37b257b-a800-4d32-9117-6f5dadd22fa8\",\n      \"name\": \"e37b257b-a800-4d32-9117-6f5dadd22fa8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T07:43:11.635376265Z\",\n      \"endTime\": \"2026-10-14T08:12:15.916382682Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-nzqkm-w8mpn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b6z74shd5q4f--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-nzqkm-w8mpn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-nzqkm-w8mpn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b6z74shd5q4f--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-nzqkm-w8mpn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0292224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e37b257b-a800-4d32-9117-6f5dadd22fa8\n            correlation ID: a458a580-f3b1-4361-8ade-4eb50ff689a4\n            client request ID: ac6a0de7-84cd-4baa-8bf7-d9a0e674cba5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a53ed53e100\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-b6z74shd5q4f\" to finish creating: GET https://rp.j0292224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e37b257b-a800-4d32-9117-6f5dadd22fa8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e37b257b-a800-4d32-9117-6f5dadd22fa8\",\n      \"name\": \"e37b257b-a800-4d32-9117-6f5dadd22fa8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T07:43:11.635376265Z\",\n      \"endTime\": \"2026-10-14T08:12:15.916382682Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-nzqkm-w8mpn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b6z74shd5q4f--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-nzqkm-w8mpn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-nzqkm-w8mpn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b6z74shd5q4f--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-nzqkm-w8mpn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0292224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e37b257b-a800-4d32-9117-6f5dadd22fa8\n            correlation ID: a458a580-f3b1-4361-8ade-4eb50ff689a4\n            client request ID: ac6a0de7-84cd-4baa-8bf7-d9a0e674cba5\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110259740570292224","occurred_at":"2026-10-14T06:41:43Z","signature_id":"fea509d928df2e88f833dc8d3fe2fd070af86fa3b0889acfe2996fef09b6fa2b","pr_number":7509,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f71af0d393321579ba6f1aaa05e8bc29e7a762e5437ac44b1d36d3dedbcacbe3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-rpntb-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-rpntb-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T11:52:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x346ef5b24080\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-xn2t4zqk2qxz\" to finish creating: GET https://rp.j6626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/439dbd7f-6ac7-4b48-8533-a8815df6b20f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/439dbd7f-6ac7-4b48-8533-a8815df6b20f\",\n      \"name\": \"439dbd7f-6ac7-4b48-8533-a8815df6b20f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T12:55:20.912229682Z\",\n      \"endTime\": \"2026-10-14T13:24:20.249835656Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-rpntb-stl8c: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-xn2t4zqk2qxz--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-rpntb-stl8c to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-rpntb-stl8c: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-xn2t4zqk2qxz--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-rpntb-stl8c to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/439dbd7f-6ac7-4b48-8533-a8815df6b20f\n            correlation ID: eefc14a4-21ec-420e-9b63-e8df580da42a\n            client request ID: b9d8fbaf-fe74-4488-8491-9a8ef5ba8fd7\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x346ef5b24080\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-xn2t4zqk2qxz\" to finish creating: GET https://rp.j6626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/439dbd7f-6ac7-4b48-8533-a8815df6b20f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/439dbd7f-6ac7-4b48-8533-a8815df6b20f\",\n      \"name\": \"439dbd7f-6ac7-4b48-8533-a8815df6b20f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T12:55:20.912229682Z\",\n      \"endTime\": \"2026-10-14T13:24:20.249835656Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-rpntb-stl8c: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-xn2t4zqk2qxz--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-rpntb-stl8c to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-rpntb-stl8c: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-xn2t4zqk2qxz--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-rpntb-stl8c to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/439dbd7f-6ac7-4b48-8533-a8815df6b20f\n            correlation ID: eefc14a4-21ec-420e-9b63-e8df580da42a\n            client request ID: b9d8fbaf-fe74-4488-8491-9a8ef5ba8fd7\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7525/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110338041116626944","occurred_at":"2026-10-14T11:52:51Z","signature_id":"e82a3bd52f8fdc258c91f1db11faa8c8e871c65a7359e7d18ab9e373fa688165","pr_number":7525,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a36cdec30df49971b11d0532fa3d90b6d04db004910c6c9ba6b1036a99dab7a3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-14T15:15:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22a4be5cfc40\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-gc2cf577p6fs\" to finish creating: GET https://rp.j0375552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1dec4678-5a13-4201-9600-24dcb5a26534\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1dec4678-5a13-4201-9600-24dcb5a26534\",\n      \"name\": \"1dec4678-5a13-4201-9600-24dcb5a26534\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T16:17:52.259133125Z\",\n      \"endTime\": \"2026-10-14T16:46:52.310381675Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-dnjw8-hs9xx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-gc2cf577p6fs--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-dnjw8-hs9xx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-dnjw8-hs9xx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-gc2cf577p6fs--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-dnjw8-hs9xx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0375552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1dec4678-5a13-4201-9600-24dcb5a26534\n            correlation ID: b81493f7-b048-4704-b54a-5a69ef8ad3b5\n            client request ID: 4dd74744-6a7e-48d1-9b1b-985d5238151a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22a4be5cfc40\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-gc2cf577p6fs\" to finish creating: GET https://rp.j0375552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1dec4678-5a13-4201-9600-24dcb5a26534\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1dec4678-5a13-4201-9600-24dcb5a26534\",\n      \"name\": \"1dec4678-5a13-4201-9600-24dcb5a26534\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T16:17:52.259133125Z\",\n      \"endTime\": \"2026-10-14T16:46:52.310381675Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-dnjw8-hs9xx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-gc2cf577p6fs--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-dnjw8-hs9xx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-dnjw8-hs9xx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-gc2cf577p6fs--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-dnjw8-hs9xx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0375552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1dec4678-5a13-4201-9600-24dcb5a26534\n            correlation ID: b81493f7-b048-4704-b54a-5a69ef8ad3b5\n            client request ID: 4dd74744-6a7e-48d1-9b1b-985d5238151a\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110389077940375552","occurred_at":"2026-10-14T15:15:39Z","signature_id":"a910098321a76c4517d5d09d23a0ef9fa638d78b5f1bc6f25a192f2153e44e26","pr_number":7492,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"95af36169869353e6e4bd033f76bffdcb34f40de57dacc0d2314155a545063e2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-fqbnw to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-qvbld to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-fqbnw to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-qvbld to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T06:47:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xcf2616b6280\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-mfxbjq8jns8s\" to finish creating: GET https://rp.j9450880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ef2bbd7-ca39-48e4-90ab-26948b5b65f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ef2bbd7-ca39-48e4-90ab-26948b5b65f1\",\n      \"name\": \"6ef2bbd7-ca39-48e4-90ab-26948b5b65f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T07:49:18.1783953Z\",\n      \"endTime\": \"2026-10-14T08:18:20.880236175Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v20261001-np-1-cw6kh-fqbnw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-fqbnw to exist\\nMachine v20261001-np-1-cw6kh-qvbld: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-qvbld to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20261001-np-1-cw6kh-fqbnw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-fqbnw to exist\\nMachine v20261001-np-1-cw6kh-qvbld: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-qvbld to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9450880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ef2bbd7-ca39-48e4-90ab-26948b5b65f1\n            correlation ID: 39af48df-821f-41b4-9eda-383af9fbc767\n            client request ID: d214e485-a699-4f3c-bb9e-2d4e1497eed9\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xcf2616b6280\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-mfxbjq8jns8s\" to finish creating: GET https://rp.j9450880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ef2bbd7-ca39-48e4-90ab-26948b5b65f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ef2bbd7-ca39-48e4-90ab-26948b5b65f1\",\n      \"name\": \"6ef2bbd7-ca39-48e4-90ab-26948b5b65f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T07:49:18.1783953Z\",\n      \"endTime\": \"2026-10-14T08:18:20.880236175Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v20261001-np-1-cw6kh-fqbnw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-fqbnw to exist\\nMachine v20261001-np-1-cw6kh-qvbld: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-qvbld to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20261001-np-1-cw6kh-fqbnw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-fqbnw to exist\\nMachine v20261001-np-1-cw6kh-qvbld: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-qvbld to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9450880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ef2bbd7-ca39-48e4-90ab-26948b5b65f1\n            correlation ID: 39af48df-821f-41b4-9eda-383af9fbc767\n            client request ID: d214e485-a699-4f3c-bb9e-2d4e1497eed9\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110261282979450880","occurred_at":"2026-10-14T06:47:51Z","signature_id":"d62d439af2428ff5da7588c1c8cdf6f1dcb51f4f80f91fc98fcda9890d13348f","pr_number":6712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"255883e0e1738fd02f2daa41d490a5ba122bcfd657ae1579f6fbfb04d4dff13d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[2,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-12"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-14T19:26:29Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-14T20:02:04.703Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 description=\"Step infra\\n  Kind: ARM\\n  Template: templates/mgmt-infra.bicep\\n  Parameters: configurations/mgmt-infra.tmpl.bicepparam\"\ntime=2026-10-14T20:02:05.214Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2\ntime=2026-10-14T20:02:05.894Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 deployment=c992d66eb9a38cb8bacf66c4633734326584f1d12bbb6c98da67de8819d3eb79 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0582272-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc992d66eb9a38cb8bacf66c4633734326584f1d12bbb6c98da67de8819d3eb79\ntime=2026-10-14T20:03:06.110Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0582272-mgmt-2/providers/Microsoft.Resources/deployments/c992d66eb9a38cb8bacf66c4633734326584f1d12bbb6c98da67de8819d3eb79/operationStatuses/08584095987599895937\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0582272-mgmt-2/providers/Microsoft.Resources/deployments/mgmt-kv-2awbxe73l63vi\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"VaultAlreadyExists\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The vault name 'ah-ci01-mg-j0582272-2' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0582272-mgmt-2/providers/Microsoft.Resources/deployments/msi-kv-g2p3wnorghzau\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"VaultAlreadyExists\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The vault name 'ah-ci01-mi-j0582272-2' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0582272-mgmt-2/providers/Microsoft.Resources/deployments/ah-ci01-me-j0582272-2\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"VaultAlreadyExists\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The vault name 'ah-ci01-me-j0582272-2' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452210310582272","occurred_at":"2026-10-14T19:26:29Z","signature_id":"d517e51fc5b09c819903bacd247416bcbbe167939922787f3a2438a125b3fdc2","pr_number":7492,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"64bb2607db9a31af645f91fcf6ffbdb70d8bca63638eea03e861ce2387c49227","failure_pattern":"ERROR CODE: DeploymentNotFound; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentNotFound","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-14T12:03:18Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-14T12:33:58.591Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-10-14T12:35:01.493Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-14T12:33:58.60113061Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5008768-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5008768-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-14T12:34:00.818434525Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5008768-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5008768-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-14T12:34:01.565464014Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5008768-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5008768-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-10-14T12:35:01.489201259Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"collecting deployment operations\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5008768-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5008768-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"err\\\":{\\\"errorCode\\\":\\\"DeploymentNotFound\\\",\\\"statusCode\\\":404,\\\"errorMessage\\\":\\\"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5008768-mgmt-2/deployments/aks-cluster-create/operations\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 404: 404 Not Found\\\\nERROR CODE: DeploymentNotFound\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentNotFound\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Deployment 'aks-cluster-create' could not be found.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"}}\\n{\\\"time\\\":\\\"2026-10-14T12:35:01.489304211Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5008768-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5008768-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"err\\\":\\\"submitting deployment aks-cluster-create: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5008768-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 400: 400 Bad Request\\\\nERROR CODE: InvalidTemplateDeployment\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"InvalidTemplateDeployment\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The template deployment 'aks-cluster-create' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.ContainerService/managedClusters (2026-04-02-preview)' reported preflight validation errors. Tracking id is 'cd5c0f42-620b-41a9-b4be-4210990b303c'. See inner errors for details.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": [\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"ListResourceSKUsFailed\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"Preflight validation check for resource(s) for container service ci01-j5008768-mgmt-2 in resource group hcp-underlay-ci01-j5008768-mgmt-2 failed. Message: List resource SKUs in subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX failed. Details: \\\\\\\"\\\\n      }\\\\n    ]\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"1m2.888193483s\\\"}\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7522/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340744895008768","occurred_at":"2026-10-14T12:03:18Z","signature_id":"9c183fffa613e8634fa50b7ecf08c849a5c8665852ba5fbee19eb46bb04baa54","pr_number":7522,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[1,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":14,"prior_last_seen_at":"2026-10-14T18:26:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x12acc23ba300\u003e: \n    failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-kwgn8mlj4cqw\" to finish creating: GET https://rp.j2922112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2922112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64\n            correlation ID: 6cf5166e-d7f7-4d42-8f52-96bfabf5f7df\n            client request ID: d4b2a1df-961f-48a7-9e13-886e82b9c210\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x12acc23ba300\u003e: \n    failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-kwgn8mlj4cqw\" to finish creating: GET https://rp.j2922112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2922112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64\n            correlation ID: 6cf5166e-d7f7-4d42-8f52-96bfabf5f7df\n            client request ID: d4b2a1df-961f-48a7-9e13-886e82b9c210\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110437162552922112","occurred_at":"2026-10-14T18:26:44Z","signature_id":"d7964fba6ca87a857bce8a35d83b03910a03ed75438c875148bf5ec27b0319d1","pr_number":7524,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c146a310e9f36bad2b2031b81dadd86ffc315b9e8d8f9b9e60f16f2e9c3f9acf","failure_pattern":"NS lookup for aro","search_query":"NS lookup for aro","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T11:15:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:394]: Timed out after 90.001s.\ndelegation and authoritative NS answers should remain consistent for at least 30 seconds before issuance\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x371ee5ea5c0\u003e: \n    NS lookup for aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud via ns4-07.azure-dns.info.: lookup aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud. on 172.30.0.10:53: dial udp: lookup ns4-07.azure-dns.info.: i/o timeout\n    {\n        msg: \"NS lookup for aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud via ns4-07.azure-dns.info.: lookup aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud. on 172.30.0.10:53: dial udp: lookup ns4-07.azure-dns.info.: i/o timeout\",\n        err: \u003c*net.DNSError | 0x371ee500500\u003e{\n            UnwrapErr: \u003c*net.OpError | 0x371ee5004b0\u003e{\n                Op: \"dial\",\n                Net: \"udp\",\n                Source: nil,\n                Addr: nil,\n                Err: \u003c*net.DNSError | 0x371ee500460\u003e{\n                    UnwrapErr: \u003c*net.timeoutError | 0x17059be0\u003e{},\n                    Err: \"i/o timeout\",\n                    Name: \"ns4-07.azure-dns.info.\",\n                    Server: \"\",\n                    IsTimeout: true,\n                    IsTemporary: true,\n                    IsNotFound: false,\n                },\n            },\n            Err: \"dial udp: lookup ns4-07.azure-dns.info.: i/o timeout\",\n            Name: \"aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud.\",\n            Server: \"172.30.0.10:53\",\n            IsTimeout: true,\n            IsTemporary: true,\n            IsNotFound: false,\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:394]: Timed out after 90.001s.\ndelegation and authoritative NS answers should remain consistent for at least 30 seconds before issuance\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x371ee5ea5c0\u003e: \n    NS lookup for aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud via ns4-07.azure-dns.info.: lookup aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud. on 172.30.0.10:53: dial udp: lookup ns4-07.azure-dns.info.: i/o timeout\n    {\n        msg: \"NS lookup for aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud via ns4-07.azure-dns.info.: lookup aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud. on 172.30.0.10:53: dial udp: lookup ns4-07.azure-dns.info.: i/o timeout\",\n        err: \u003c*net.DNSError | 0x371ee500500\u003e{\n            UnwrapErr: \u003c*net.OpError | 0x371ee5004b0\u003e{\n                Op: \"dial\",\n                Net: \"udp\",\n                Source: nil,\n                Addr: nil,\n                Err: \u003c*net.DNSError | 0x371ee500460\u003e{\n                    UnwrapErr: \u003c*net.timeoutError | 0x17059be0\u003e{},\n                    Err: \"i/o timeout\",\n                    Name: \"ns4-07.azure-dns.info.\",\n                    Server: \"\",\n                    IsTimeout: true,\n                    IsTemporary: true,\n                    IsNotFound: false,\n                },\n            },\n            Err: \"dial udp: lookup ns4-07.azure-dns.info.: i/o timeout\",\n            Name: \"aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud.\",\n            Server: \"172.30.0.10:53\",\n            IsTimeout: true,\n            IsTemporary: true,\n            IsNotFound: false,\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110328620604985344","occurred_at":"2026-10-14T11:15:25Z","signature_id":"5d8b5b70f0a9c1c8487aba6b0baf30179d57edeb456ee14fadec7eb5d56e4327","pr_number":7524,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":9,"prior_last_seen_at":"2026-10-14T18:32:20Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x8382000a060\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x8381fe06080\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x8381fe064a0\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x8381fe06480\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x8382000a060\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x8381fe06080\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x8381fe064a0\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x8381fe06480\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7528/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110438573613256704","occurred_at":"2026-10-14T18:32:20Z","signature_id":"ad373b5f327b0c9377a01e8a308f067975dee30f288b975f964b13f502d4e9ed","pr_number":7528,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7a87df562eaa8a0bfda8c14840c36e7197e35a3b92f22e4f536b1e3d2ed289ff","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-12"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-15T05:42:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x18c85c0bedc8\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x18c85ad32dc0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x18c85bf9a260\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x18c85bf9a240\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x18c85c0bedc8\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x18c85ad32dc0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x18c85bf9a260\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x18c85bf9a240\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7487/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110607153617178624","occurred_at":"2026-10-15T05:42:13Z","signature_id":"81b57b07ae09640e6ee1ce4f8b6361b066bdec840190e2be0ddc6c4048104564","pr_number":7487,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8e7e1939a67145fe288ccb305762ce0b966b7e2b9d52d9aa38f5a043a3e3e9f2","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: SWIFT query row","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T00:18:45.8286809Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\", ActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", SubActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityStack:\"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f \u003e GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\"}, IsPermanent:false})","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T23:00:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f63e25da900\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T00:18:45.8286809Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\", ActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", SubActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityStack:\"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f \u003e GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T00:18:45.8286809Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\\\", ActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", SubActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f \u003e GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x1f63e28703a0\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T00:18:45.8286809Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\\\", ActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", SubActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f \u003e GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x1f63e2d64800\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-15T00:18:45.8286809Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\",\n                        ActivityId: \"3e4bf140-f035-443b-96b4-bed0d927cd4f\",\n                        SubActivityId: \"3e4bf140-f035-443b-96b4-bed0d927cd4f\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"3e4bf140-f035-443b-96b4-bed0d927cd4f\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f \u003e GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f63e25da900\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T00:18:45.8286809Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\", ActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", SubActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityStack:\"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f \u003e GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T00:18:45.8286809Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\\\", ActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", SubActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f \u003e GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x1f63e28703a0\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T00:18:45.8286809Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\\\", ActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", SubActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f \u003e GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x1f63e2d64800\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-15T00:18:45.8286809Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\",\n                        ActivityId: \"3e4bf140-f035-443b-96b4-bed0d927cd4f\",\n                        SubActivityId: \"3e4bf140-f035-443b-96b4-bed0d927cd4f\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"3e4bf140-f035-443b-96b4-bed0d927cd4f\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f \u003e GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110505977575903232","occurred_at":"2026-10-14T23:00:10Z","signature_id":"dd27870673151fbc796c3edce7816006c004f55fff53275326afd3ff6f2ddb19","pr_number":7451,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4291ee8f6243a25b8eeb49236e9e84795ced38f439b9996bfab09d4200eb4d6c","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in \u003curl\u003e context deadline exceeded","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T11:52:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2620bace6480\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2620bace6460\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2620bace6480\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2620bace6460\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7525/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110338041116626944","occurred_at":"2026-10-14T11:52:51Z","signature_id":"e02e171a721a96c3dc0dd8058864c72da7053641616de4d9a79e76ea8f3ae577","pr_number":7525,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"45d0345406d5a551963c23a5ded3db1272e39d63a6c7ca06460fb46b276ed104","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-15T01:16:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lq67n7wxjv9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xa4868a84700\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lq67n7wxjv9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-2teu677bb6kl3jnq3h6qj1u9ribq025b-z3n3z0e0b6i7l6i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lq67n7wxjv9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-2teu677bb6kl3jnq3h6qj1u9ribq025b-z3n3z0e0b6i7l6i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0xa4869d391c0\u003e{\n            s: \"dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-2teu677bb6kl3jnq3h6qj1u9ribq025b-z3n3z0e0b6i7l6i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lq67n7wxjv9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xa4868a84700\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lq67n7wxjv9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-2teu677bb6kl3jnq3h6qj1u9ribq025b-z3n3z0e0b6i7l6i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lq67n7wxjv9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-2teu677bb6kl3jnq3h6qj1u9ribq025b-z3n3z0e0b6i7l6i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0xa4869d391c0\u003e{\n            s: \"dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-2teu677bb6kl3jnq3h6qj1u9ribq025b-z3n3z0e0b6i7l6i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7540/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110540376279355392","occurred_at":"2026-10-15T01:16:52Z","signature_id":"afc5dc16004fe00ec4fadfcd21a3566176869acceef40b347c6fdeba6cb70b36","pr_number":7540,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1fb02875481a82a322575aa62ae21fd2ca87b013af435787776c9187994426c8","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-15T03:25:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tzcs2w52g775/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x6f640d28320\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tzcs2w52g775/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-2tf0946f9b41h1n7hltqnq95f64jvhk8-j9h5p7q3y7t6s2u: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tzcs2w52g775/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-2tf0946f9b41h1n7hltqnq95f64jvhk8-j9h5p7q3y7t6s2u: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x6f64075f4f0\u003e{\n            s: \"dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-2tf0946f9b41h1n7hltqnq95f64jvhk8-j9h5p7q3y7t6s2u: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tzcs2w52g775/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x6f640d28320\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tzcs2w52g775/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-2tf0946f9b41h1n7hltqnq95f64jvhk8-j9h5p7q3y7t6s2u: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tzcs2w52g775/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-2tf0946f9b41h1n7hltqnq95f64jvhk8-j9h5p7q3y7t6s2u: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x6f64075f4f0\u003e{\n            s: \"dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-2tf0946f9b41h1n7hltqnq95f64jvhk8-j9h5p7q3y7t6s2u: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7401/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110572658033692672","occurred_at":"2026-10-15T03:25:08Z","signature_id":"079d9e11ecf4c9bf783efc15c5463fb2619c071c1ffcb6b29c1c7dce8abd75ae","pr_number":7401,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6f5b0699b6f1437b23a1c7da6f46df6d98e827812483320febb57823f35e33eb","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in \u003curl\u003e context deadline exceeded","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-15T02:28:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3527505f6640\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3527505f6620\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3527505f6640\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3527505f6620\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558398402531328","occurred_at":"2026-10-15T02:28:29Z","signature_id":"7361959de99cf2e3597ad30ee7670a2daf3f64f017ca0d0ae2cd989364d45570","pr_number":7497,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0d1bb767dc6bcd521c34a14e9b0517517350a11dc10d0bf7b970c9a0d6ce4727","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: SWIFT query row","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:29.6322749Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\", ActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", SubActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd \u003e GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\"}, IsPermanent:false})","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T09:30:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x21de3cbea400\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:29.6322749Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\", ActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", SubActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd \u003e GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T11:13:29.6322749Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\\\", ActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", SubActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd \u003e GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x21de3ca9e560\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T11:13:29.6322749Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\\\", ActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", SubActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd \u003e GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x21de3e2c5500\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-14T11:13:29.6322749Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\",\n                        ActivityId: \"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\",\n                        SubActivityId: \"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd \u003e GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x21de3cbea400\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:29.6322749Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\", ActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", SubActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd \u003e GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T11:13:29.6322749Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\\\", ActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", SubActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd \u003e GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x21de3ca9e560\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T11:13:29.6322749Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\\\", ActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", SubActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd \u003e GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x21de3e2c5500\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-14T11:13:29.6322749Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\",\n                        ActivityId: \"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\",\n                        SubActivityId: \"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd \u003e GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7522/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110302904370860032","occurred_at":"2026-10-14T09:30:21Z","signature_id":"28ee968824b537cbd60af98cfb420def9e8a07138c1b26600fb5f40c37a6e2be","pr_number":7522,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4bdce0fc0263ab649c57301068b1470e004ac35ea6a6dc4e6dec1a88f568ae7f","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in \u003curl\u003e context deadline exceeded","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T09:25:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xee59296c020\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0xee59296c000\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xee59296c020\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0xee59296c000\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7523/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110300965679665152","occurred_at":"2026-10-14T09:25:32Z","signature_id":"2b1ef112e3fe566ad64f97a08cb37e00dd8e121aa32253ddb9ce1f4e565230f3","pr_number":7523,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ace3655e8c7da2e020456d9d99c4eaa589f6c874efd0e56cdc422ea75791aa32","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in \u003curl\u003e context deadline exceeded","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T19:26:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ba1a496c2e0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2ba1a496c2c0\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ba1a496c2e0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2ba1a496c2c0\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452244770983936","occurred_at":"2026-10-14T19:26:40Z","signature_id":"b0b4717112a8ce5606c552fa6abfbc35d74e436e05d5aea424433af3362f5a86","pr_number":7509,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c9083ec9c855a1c2a34a116ed4f451e20e6042d364acfc229b5b5f3aa5fba483","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in \u003curl\u003e context deadline exceeded","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T19:26:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x298032100120\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x298032100100\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x298032100120\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x298032100100\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7499/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452208637054976","occurred_at":"2026-10-14T19:26:26Z","signature_id":"d6ddfa46d3d745e923af613cb97a80b523ed08d2ae0ec6f2e9ddf174c80c66c2","pr_number":7499,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d44d2c4fe704ed3c143d81a48596157d54af9a75f3ee6b16e677749a6ef178d6","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: SWIFT query row","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:43.5550944Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\", ActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", SubActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityStack:\"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 \u003e GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\"}, IsPermanent:false})","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T09:37:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1dded76126c0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:43.5550944Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\", ActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", SubActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityStack:\"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 \u003e GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T11:13:43.5550944Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\\\", ActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", SubActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 \u003e GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x1dded76126a0\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T11:13:43.5550944Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\\\", ActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", SubActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 \u003e GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x1dded776cc00\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-14T11:13:43.5550944Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\",\n                        ActivityId: \"0eb2baf5-e792-455b-9c86-a0654265ab17\",\n                        SubActivityId: \"0eb2baf5-e792-455b-9c86-a0654265ab17\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"0eb2baf5-e792-455b-9c86-a0654265ab17\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 \u003e GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1dded76126c0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:43.5550944Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\", ActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", SubActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityStack:\"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 \u003e GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T11:13:43.5550944Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\\\", ActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", SubActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 \u003e GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x1dded76126a0\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T11:13:43.5550944Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\\\", ActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", SubActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 \u003e GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x1dded776cc00\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-14T11:13:43.5550944Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\",\n                        ActivityId: \"0eb2baf5-e792-455b-9c86-a0654265ab17\",\n                        SubActivityId: \"0eb2baf5-e792-455b-9c86-a0654265ab17\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"0eb2baf5-e792-455b-9c86-a0654265ab17\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 \u003e GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110303886584582144","occurred_at":"2026-10-14T09:37:08Z","signature_id":"2f0e6a655ab2ba2b3047dea03f9db87c620221457bccc3eb57eb415eeee14c00","pr_number":7383,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c7fba217b6ed5ce66bf9da85a1ad0fe4ef234d9b8e28c1d62c1a68359ce05dd8","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in \u003curl\u003e context deadline exceeded","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T09:43:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x128374ba040\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x128374ba020\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x128374ba040\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x128374ba020\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7498/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110305441572458496","occurred_at":"2026-10-14T09:43:19Z","signature_id":"7df74c111616fc166a1f2bc4df7a5d0c152bae783e433e710f514e16483b6958","pr_number":7498,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"dfd71f6e4c7fe588ab399e6a472c132eceea6fa6ee6f4e3aad318dc1c9c07df2","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected all","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-15T04:29:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w6xbc82hxr2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x280af577a000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w6xbc82hxr2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-2tf1898kdmei2jv6trajehsfpj8o4bk2-f7r5q8n1l4e4d9o: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w6xbc82hxr2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-2tf1898kdmei2jv6trajehsfpj8o4bk2-f7r5q8n1l4e4d9o: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0x280af63caa80\u003e{\n            s: \"dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-2tf1898kdmei2jv6trajehsfpj8o4bk2-f7r5q8n1l4e4d9o: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w6xbc82hxr2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x280af577a000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w6xbc82hxr2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-2tf1898kdmei2jv6trajehsfpj8o4bk2-f7r5q8n1l4e4d9o: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w6xbc82hxr2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-2tf1898kdmei2jv6trajehsfpj8o4bk2-f7r5q8n1l4e4d9o: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0x280af63caa80\u003e{\n            s: \"dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-2tf1898kdmei2jv6trajehsfpj8o4bk2-f7r5q8n1l4e4d9o: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110588894188670976","occurred_at":"2026-10-15T04:29:39Z","signature_id":"5b412609e3463a02d7ca75b01131987fcae35acf585438d7204d1a174f6575bc","pr_number":6746,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a754bbeaf568a9f11e5562cb750f138c5a1e5121c5f07c9c3464251c13a17fc9","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in \u003curl\u003e context deadline exceeded","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T19:26:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32bc856472e0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x32bc85647260\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32bc856472e0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x32bc85647260\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452257878183936","occurred_at":"2026-10-14T19:26:43Z","signature_id":"befc5c1e01c5061c2aad6d5d0c355f744c533fb274c0bab64d76e7eb496cca3b","pr_number":6746,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9f75b69b94f5e69dd2ea89d590d15c51614ce3eda1965d4efc965f1caf5f8e2b","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption Burst High - \u003ccosmos-account\u003e fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption Burst High - \u003ccosmos-account\u003e fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption Burst High - arohcpci01-rp-j7369856 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[3,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-14T02:41:42Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption Burst High - arohcpci01-rp-j7369856 does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-14T03:40:18Z\n  Ended:       2026-10-14T03:43:19Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 95% averaged over a 5-minute window, evaluated every minute. This catches short RU-saturation bursts (e.g. mass cluster creation at the start of E2E tests) that exceed provisioned throughput and can lead to 429 throttling before the 10-minute sustained alert fires. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-14T03:52:13Z\n  Ended:       2026-10-14T04:06:24Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 95% averaged over a 5-minute window, evaluated every minute. This catches short RU-saturation bursts (e.g. mass cluster creation at the start of E2E tests) that exceed provisioned throughput and can lead to 429 throttling before the 10-minute sustained alert fires. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110199338117369856","occurred_at":"2026-10-14T02:41:42Z","signature_id":"517b35de6d4d5ec7325035eafd6825972465809bddcc7128c935e22b8d7cb4df","pr_number":7495,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4275a314d89bf06a724f7bf3c5bfe06406a90fa00482d282b46f40a1b17ff273","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7369856 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[4,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-14T02:41:42Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7369856 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-14T03:48:00Z\n  Ended:       2026-10-14T04:18:05Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 15-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110199338117369856","occurred_at":"2026-10-14T02:41:42Z","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","pr_number":7495,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[1,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":31,"prior_last_seen_at":"2026-10-14T19:26:58Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-14T20:45:07Z\n  Ended:       2026-10-14T20:56:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j2147200-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j2147200-svc).\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-14T21:06:15Z\n  Ended:       2026-10-14T21:18:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j2147200-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j2147200-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452322852147200","occurred_at":"2026-10-14T19:26:58Z","signature_id":"823d75e8489768f41c2c16693a58fb2d5bd13ee5dfe3292082f19eb7aee114da","pr_number":7497,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1650432ef7f4a895179256b7f0a1a789aa6cd26091bc9f9d60b8d69099e131bf","failure_pattern":"alert [svc] KubeJobFailed fired","search_query":"alert [svc] KubeJobFailed fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeJobFailed does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T14:05:01Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeJobFailed does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T14:59:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeJobFailed\", cluster=\"ci01-j4159872-svc\", component=\"kubernetes-infrastructure\", condition=\"true\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.26:8080\", job=\"kube-state-metrics\", job_name=\"hubble-generate-certs-55c4c87c\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4159872/providers/microsoft.monitor/accounts/services-j4159872\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-mgfnh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Job kube-system/hubble-generate-certs-55c4c87c failed to complete. Removing failed job after investigation should clear this alert.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T14:59:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeJobFailed\", cluster=\"ci01-j4159872-svc\", component=\"kubernetes-infrastructure\", condition=\"true\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.26:8080\", job=\"kube-state-metrics\", job_name=\"hubble-generate-certs-55c4c87c\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4159872/providers/microsoft.monitor/accounts/services-j4159872\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-mgfnh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Job kube-system/hubble-generate-certs-55c4c87c failed to complete. Removing failed job after investigation should clear this alert."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7528/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110371307534159872","occurred_at":"2026-10-14T14:05:01Z","signature_id":"16c569152aa137a42851185754971c002c5a0136da3e260a036f60238b7422d2","pr_number":7528,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[1,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":53,"prior_last_seen_at":"2026-10-14T17:09:29Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":1}],"full_error_samples":["alert fired 70 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-4fnrs\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-4fnrs has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-qd4lc\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-qd4lc has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-q9htv\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-q9htv has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-2v2ll\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-2v2ll has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-p45vd\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-p45vd has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-v2bxl\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-v2bxl has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-rsvx7\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-rsvx7 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-zw6fj\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-zw6fj has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-r5r7n\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-r5r7n has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-xb9hp\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-xb9hp has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-bvr8s\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-bvr8s has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-vsrr8\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-vsrr8 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-srv48\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-srv48 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-q5vds\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-q5vds has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-rjwtc\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-rjwtc has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-nzgpk\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-nzgpk has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-4slr2\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-4slr2 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-m7ftv\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-m7ftv has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-v7hdm\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-v7hdm has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-86ssh\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-86ssh has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-jp6fj\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-jp6fj has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-xjxzz\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-xjxzz has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-b7ph9\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-b7ph9 has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-drpfk\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-drpfk has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-6q6w9\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-6q6w9 has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-77r8t\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-77r8t has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-vhdzq\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-vhdzq has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-hdcnl\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-hdcnl has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-5z589\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-5z589 has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-kr492\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-kr492 has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-bcl97\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-bcl97 has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-s49mm\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-s49mm has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-drmdx\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-drmdx has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-sm8gg\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-sm8gg has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-2xnc2\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-2xnc2 has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-7x9gw\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-7x9gw has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-bddnr\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-bddnr has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-kz42p\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-kz42p has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-nz88g\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-nz88g has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-4pq7x\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-4pq7x has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-flmn9\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-flmn9 has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-fwvds\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-fwvds has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-mk9q7\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-mk9q7 has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-6bqxw\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-6bqxw has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-xn8q4\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-xn8q4 has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-4s44z\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-4s44z has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-fhhv2\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-fhhv2 has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-t8v8f\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-t8v8f has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-2sg4p\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-2sg4p has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-mh74p\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-mh74p has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-fgn4b\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-fgn4b has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-lq8md\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-lq8md has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-wcdz6\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-wcdz6 has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-qbfw5\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-qbfw5 has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-nshz9\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-nshz9 has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-f6nsp\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-f6nsp has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-z6t2j\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-z6t2j has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-s75xf\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-s75xf has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-bmx5q\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-bmx5q has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-t57qd\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-t57qd has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-4ntcs\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-4ntcs has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-cjtgq\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-cjtgq has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-6jl7m\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-6jl7m has been in a non-ready state for longer than 5 minutes.\nFiring 64:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-79sxk\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-79sxk has been in a non-ready state for longer than 5 minutes.\nFiring 65:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-pqnwt\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-pqnwt has been in a non-ready state for longer than 5 minutes.\nFiring 66:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-hplkm\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-hplkm has been in a non-ready state for longer than 5 minutes.\nFiring 67:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-drfrc\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-drfrc has been in a non-ready state for longer than 5 minutes.\nFiring 68:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-9k4dd\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-9k4dd has been in a non-ready state for longer than 5 minutes.\nFiring 69:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-4zv7k\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-4zv7k has been in a non-ready state for longer than 5 minutes.\nFiring 70:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T17:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6227072-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-ql2fg\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-5ddf5fbfd4-ql2fg has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110417722436227072","occurred_at":"2026-10-14T17:09:29Z","signature_id":"7a7c5e86b9ade1b9e5367692230bf2de020f056cf10152974063cca4acfddab3","pr_number":7519,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"599d126c4f18389701726c36f576de53996cc8bf35c7455f30ca19957cc25b45","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"3vAWYLSpgMpdmMCKnkDG1+GD7QF9y2L9E9AOfsGsY04=\"","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"3vAWYLSpgMpdmMCKnkDG1+GD7QF9y2L9E9AOfsGsY04=\"","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T15:15:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:364]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x181b25f73380\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"3vAWYLSpgMpdmMCKnkDG1+GD7QF9y2L9E9AOfsGsY04=\"\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \\\"3vAWYLSpgMpdmMCKnkDG1+GD7QF9y2L9E9AOfsGsY04=\\\"\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:364]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x181b25f73380\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"3vAWYLSpgMpdmMCKnkDG1+GD7QF9y2L9E9AOfsGsY04=\"\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \\\"3vAWYLSpgMpdmMCKnkDG1+GD7QF9y2L9E9AOfsGsY04=\\\"\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110389003348873216","occurred_at":"2026-10-14T15:15:22Z","signature_id":"586406f9efd0bd6dcafe1c3adb0dee6382f6f3192b935a25b350e0fba5d752ea","pr_number":7500,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[2,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":12,"prior_last_seen_at":"2026-10-14T21:30:12Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:364]: Timed out after 300.007s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3db417569280\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:54452-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:54452-\u003e172.30.0.10:53: read: connection refused\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:364]: Timed out after 300.007s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3db417569280\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:54452-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:54452-\u003e172.30.0.10:53: read: connection refused\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110483335657558016","occurred_at":"2026-10-14T21:30:12Z","signature_id":"87f77c6532d69c77561fa791550c191175a258371f754dfc003a6b27e5e2704d","pr_number":7524,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4d876456a841781d4034f0b3b80449cf5b52a93b8aafda5fb51135d2c9b1e459","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j2981888-mgmt-1\",\"clusterName\":\"ci01-j298188","search_query":"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-14T19:33:50.055321564Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2981888-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2981888-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-14T19:33:52.298968498Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2981888-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2981888-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-14T19:33:52.677544841Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2981888-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2981888-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T18:45:15Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-10-14T19:33:50.044Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-10-14T20:03:49.640Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-14T19:33:50.055321564Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2981888-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2981888-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-14T19:33:52.298968498Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2981888-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2981888-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-14T19:33:52.677544841Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2981888-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2981888-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6946/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110441834562981888","occurred_at":"2026-10-14T18:45:15Z","signature_id":"3413e51414076b21936c915303e7fa16ef8b8cf4ecc8fff350a16108a153e807","pr_number":6946,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cca9ba181eaef5e426cb8fe33160e83ad67f9d053e87b16b9d8d8cbf3c5cc14d","failure_pattern":"failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs","search_query":"failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T06:47:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 602.098s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3618a9570d00\u003e: \n    failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\n    | where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\n    | distinct cid, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": net/http: TLS handshake timeout\n    {\n        msg: \"failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n        err: \u003c*fmt.wrapError | 0x3618a9570cc0\u003e{\n            msg: \"failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n            err: \u003c*fmt.wrapError | 0x3618a9570c60\u003e{\n                msg: \"failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n                err: \u003c*fmt.wrapError | 0x3618a9570c40\u003e{\n                    msg: \"failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n                    err: \u003c*errors.Error | 0x3618a9654280\u003e{\n                        Op: 1,\n                        Kind: 7,\n                        Err: \u003c*fmt.wrapError | 0x3618a9570c00\u003e{\n                            msg: \"With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n                            err: \u003c*url.Error | 0x3618aa1308a0\u003e{\n                                Op: \"Post\",\n                                URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                                Err: \u003chttp.tlsHandshakeTimeoutError\u003e{},\n                            },\n                        },\n                        restErrMsg: nil,\n                        decoded: nil,\n                        permanent: false,\n                        inner: nil,\n                    },\n                },\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 602.098s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3618a9570d00\u003e: \n    failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\n    | where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\n    | distinct cid, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": net/http: TLS handshake timeout\n    {\n        msg: \"failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n        err: \u003c*fmt.wrapError | 0x3618a9570cc0\u003e{\n            msg: \"failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n            err: \u003c*fmt.wrapError | 0x3618a9570c60\u003e{\n                msg: \"failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n                err: \u003c*fmt.wrapError | 0x3618a9570c40\u003e{\n                    msg: \"failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n                    err: \u003c*errors.Error | 0x3618a9654280\u003e{\n                        Op: 1,\n                        Kind: 7,\n                        Err: \u003c*fmt.wrapError | 0x3618a9570c00\u003e{\n                            msg: \"With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n                            err: \u003c*url.Error | 0x3618aa1308a0\u003e{\n                                Op: \"Post\",\n                                URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                                Err: \u003chttp.tlsHandshakeTimeoutError\u003e{},\n                            },\n                        },\n                        restErrMsg: nil,\n                        decoded: nil,\n                        permanent: false,\n                        inner: nil,\n                    },\n                },\n            },\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110261282979450880","occurred_at":"2026-10-14T06:47:51Z","signature_id":"d94811361b1b2c494889c4f4d830e8abab41778eec15e79fdb85a81df8ec3ea3","pr_number":6712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"38232e514c86bfe6cab965d283fee05c3ce21024c2a543efb679b24a42cf1009","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T00:39:37Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1e936bab0320\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\n    let _endTime = datetime(2026-10-14T02:13:05.3130233Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-zxm6njn97hd4'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T02:13:17.5036831Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\", ActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", SubActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x1e936b180db0\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x1e936bab0300\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x1e936bab02e0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x1e93699ddd20\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x1e93699ddc60\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x1e936a2c1400\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1e936bab0320\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\n    let _endTime = datetime(2026-10-14T02:13:05.3130233Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-zxm6njn97hd4'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T02:13:17.5036831Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\", ActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", SubActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x1e936b180db0\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x1e936bab0300\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x1e936bab02e0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x1e93699ddd20\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x1e93699ddc60\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x1e936a2c1400\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110168614748295168","occurred_at":"2026-10-14T00:39:37Z","signature_id":"83eb0b332787f68fc85dd200d810732ea20732bc14eafa3beefcaaf8deb483c0","pr_number":7509,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b6abc857fe5890451778a1e8bd584c9cdff0daecc370ddea658871d7ad261b7b","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T02:04:28Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1fce6bc96f40\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\n    let _endTime = datetime(2026-10-14T03:48:07.3491058Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-sbj59nsxlhq5'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T03:48:19.7777147Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\", ActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", SubActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x1fce6bb2e558\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x1fce6bc96f20\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x1fce6bc96f00\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x1fce6b3e4d40\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x1fce6b3e4d00\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x1fce6d014600\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1fce6bc96f40\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\n    let _endTime = datetime(2026-10-14T03:48:07.3491058Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-sbj59nsxlhq5'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T03:48:19.7777147Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\", ActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", SubActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x1fce6bb2e558\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x1fce6bc96f20\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x1fce6bc96f00\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x1fce6b3e4d40\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x1fce6b3e4d00\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x1fce6d014600\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7510/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110189966687473664","occurred_at":"2026-10-14T02:04:28Z","signature_id":"a63d52fcb7a18aef09e6286c67a8d5f921df82fc1ef1c88cc7261cf7af793684","pr_number":7510,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7db277832781bbf28aea2476274ab473a26eb0485a79733307a4e7919213f9d8","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T18:22:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x356d91091ca0\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\n    let _endTime = datetime(2026-10-14T19:48:15.7310678Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T19:48:28.4804395Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\", ActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", SubActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x356d921ef170\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x356d91091c80\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x356d91091c60\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x356d91091b60\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x356d91091b20\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x356d9135a600\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x356d91091ca0\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\n    let _endTime = datetime(2026-10-14T19:48:15.7310678Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T19:48:28.4804395Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\", ActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", SubActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x356d921ef170\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x356d91091c80\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x356d91091c60\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x356d91091b60\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x356d91091b20\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x356d9135a600\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110436010226618368","occurred_at":"2026-10-14T18:22:06Z","signature_id":"79ceb6ec46d015e2d8448735d45968be712ed99bbb1e9bded870eb184f6c9787","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"09c6c22177bc75493a0fe4d10381237aa66c36f43bae5c0efd353ae0bc7fe378","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T21:30:12Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 611.066s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x32cad0a314a0\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\n    let _endTime = datetime(2026-10-14T23:11:15.1642696Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-lznjhs5kd54t'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T23:11:29.6178254Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\", ActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", SubActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x32cad17d37b8\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x32cad0a31480\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x32cad0a31460\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x32cad0a313a0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x32cad0a31360\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x32cacfbb4000\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 611.066s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x32cad0a314a0\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\n    let _endTime = datetime(2026-10-14T23:11:15.1642696Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-lznjhs5kd54t'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T23:11:29.6178254Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\", ActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", SubActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x32cad17d37b8\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x32cad0a31480\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x32cad0a31460\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x32cad0a313a0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x32cad0a31360\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x32cacfbb4000\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110483335657558016","occurred_at":"2026-10-14T21:30:12Z","signature_id":"9f1df4172af337452f2016a62aa2acc119ed3772d4f79dec7f55d22bea8822df","pr_number":7524,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"defe5d468c26c44c499810219b81a1dcd05b142f28a898794885f56e139df072","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-14T15:52:39Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-14T16:56:13.797Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 description=\"Step deploy-mce-config\\n Kind: Helm\\n\"\ntime=2026-10-14T16:56:14.254Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157 global:map[imageOverrides:map[governance_policy_propagator:governance-policy-propagator-acm-216@sha256:deb737b19932d520553c902ee5e0738ebeaad4592e8aa9649c41fec6fc00b62e] registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-10-14T16:56:14.254Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2\ntime=2026-10-14T16:56:14.292Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 namespace=multicluster-engine\ntime=2026-10-14T16:56:14.362Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 dryRun=false\ntime=2026-10-14T16:56:14.362Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2\ntime=2026-10-14T16:56:14.432Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2\ntime=2026-10-14T17:06:42.230Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 err=\"failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\"\ntime=2026-10-14T17:06:42.231Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110398398338174976","occurred_at":"2026-10-14T15:52:39Z","signature_id":"4fbcbcae7d0f352f867128831decf8a2cd0c0541228c3392fa864aa36c4d6550","pr_number":7509,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"69cfaa9caed7e22dd6863796a2f42fb482942befe7200b2f0caeda831337f912","failure_pattern":"waiting for a presented DNS-01 challenge for *","search_query":"waiting for a presented DNS-01 challenge for *","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.7142857142857143,"anchor_occurrences":1,"after_last_push_count":0,"also_in":["PROD"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T23:01:28Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:444]: Timed out after 90.001s.\ndelegated NS and ACME TXT should resolve consistently on all authoritative nameservers, or the certificate should already be issued\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2cc845a2b820\u003e: \n    waiting for a presented DNS-01 challenge for *.apps.aro.ingress-cert-le.p7kd.j2563328.hcp.osadev.cloud\n    {\n        s: \"waiting for a presented DNS-01 challenge for *.apps.aro.ingress-cert-le.p7kd.j2563328.hcp.osadev.cloud\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:444]: Timed out after 90.001s.\ndelegated NS and ACME TXT should resolve consistently on all authoritative nameservers, or the certificate should already be issued\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2cc845a2b820\u003e: \n    waiting for a presented DNS-01 challenge for *.apps.aro.ingress-cert-le.p7kd.j2563328.hcp.osadev.cloud\n    {\n        s: \"waiting for a presented DNS-01 challenge for *.apps.aro.ingress-cert-le.p7kd.j2563328.hcp.osadev.cloud\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7416/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110506304622563328","occurred_at":"2026-10-14T23:01:28Z","signature_id":"c83040805ddaac240d23b90d241ffdf180ac51f18dd8ed271efd3fd79b2f2191","pr_number":7416,"after_last_push_of_merged_pr":false}]}]},{"environment":"prod","summary":{"total_runs":5,"failed_runs":5,"raw_occurrences":13,"matched_occurrences":13,"runs_affected":5},"rows":[{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":5,"after_last_push_count":60,"also_in":["STG"],"trend_counts":[5,5,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":22,"prior_last_seen_at":"2026-10-14T23:00:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1f3c2b4a7320\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-rdjxgcpqq89z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-rdjxgcpqq89z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n            correlation ID: 7d77a59e-13ec-4222-9731-91c8b7391fdc\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1f3c2b4a7320\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-rdjxgcpqq89z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-rdjxgcpqq89z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n            correlation ID: 7d77a59e-13ec-4222-9731-91c8b7391fdc\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2e90f118a870\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-bc9b2mq7j2fk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-bc9b2mq7j2fk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n            correlation ID: 3b96e01c-3f99-420c-b36e-0060176be840\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2e90f118a870\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-bc9b2mq7j2fk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-bc9b2mq7j2fk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n            correlation ID: 3b96e01c-3f99-420c-b36e-0060176be840\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:198]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x37edba361e60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-dfl62znqdqk2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-dfl62znqdqk2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n            correlation ID: e614afff-dbc6-4d8e-b107-e0791f585e4e\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:198]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x37edba361e60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-dfl62znqdqk2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-dfl62znqdqk2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n            correlation ID: e614afff-dbc6-4d8e-b107-e0791f585e4e\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110505995661742080","occurred_at":"2026-10-14T23:00:15Z","signature_id":"04c4ff9240bfc96328974d8fef05c048d5d9bf97339039ac23a5904e5d2f1192","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110505995661742080","occurred_at":"2026-10-14T23:00:15Z","signature_id":"0a7f872dfbda04f97e316c77c7516ead2a6e83d1d2c0a4303791d99981dad0c3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110505995661742080","occurred_at":"2026-10-14T23:00:15Z","signature_id":"2b1c630a8187adc4f7f91dc9a28db83c70e788e7789784dc7c19973aba7a8c0c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110505995661742080","occurred_at":"2026-10-14T23:00:15Z","signature_id":"c4fd6cb7dad1898836db3bfad3553caa39edb467ad0049c93189b74d3c389531","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110505995661742080","occurred_at":"2026-10-14T23:00:15Z","signature_id":"fc548b2bf6881c0c3f143ff03e6b6d2641792df41afacd12b79336d46c1411aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f1386f529db6e94f57caba5baeea25a7ed546b620757afe7e7438f6ec564ee32","failure_pattern":"CreateOrUpdateExternalAuthAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-4swfh9bhdjlh\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-4swfh9bhdjlh, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":60,"anchor_occurrences":3,"after_last_push_count":3,"also_in":["STG"],"trend_counts":[0,2,1,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-15T02:00:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:262]: failed to apply external auth config with prefix policy None\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x1d3b45812a50\u003e: \n    failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-5lx5864xtdrb\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-5lx5864xtdrb, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-5lx5864xtdrb\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-5lx5864xtdrb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x1d3b4591ca00\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-5lx5864xtdrb\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:262]: failed to apply external auth config with prefix policy None\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x1d3b45812a50\u003e: \n    failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-5lx5864xtdrb\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-5lx5864xtdrb, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-5lx5864xtdrb\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-5lx5864xtdrb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x1d3b4591ca00\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-5lx5864xtdrb\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:262]: failed to apply external auth config with prefix policy None\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x2be35a9dc6c0\u003e: \n    failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-4swfh9bhdjlh\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-4swfh9bhdjlh, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-4swfh9bhdjlh\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-4swfh9bhdjlh, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x2be35bcc7190\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-4swfh9bhdjlh\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:262]: failed to apply external auth config with prefix policy None\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x2be35a9dc6c0\u003e: \n    failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-4swfh9bhdjlh\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-4swfh9bhdjlh, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-4swfh9bhdjlh\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-4swfh9bhdjlh, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x2be35bcc7190\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-4swfh9bhdjlh\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:262]: failed to apply external auth config with prefix policy None\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x130faa8f320\u003e: \n    failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-bkm7lwv87ztr\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bkm7lwv87ztr, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-bkm7lwv87ztr\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bkm7lwv87ztr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x130f942f8e0\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bkm7lwv87ztr\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:262]: failed to apply external auth config with prefix policy None\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x130faa8f320\u003e: \n    failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-bkm7lwv87ztr\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bkm7lwv87ztr, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-bkm7lwv87ztr\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bkm7lwv87ztr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x130f942f8e0\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bkm7lwv87ztr\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2110551294769893376","occurred_at":"2026-10-15T02:00:15Z","signature_id":"c13c3e65e9efeaaa2e6e039b31313b311a3206f0767adab56ba23495159e1fb6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110505995661742080","occurred_at":"2026-10-14T23:00:15Z","signature_id":"8e0d229e8c273cfdb2618315909e8aec4ed7980e908455d60fe86b2cbd902c92","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2110189368546168832","occurred_at":"2026-10-14T02:00:55Z","signature_id":"3e20acc848c2269da50d9c76e9d94a46ebbb03e0c5238076f79b8df8ef6b3923","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f60b238f5515effd82f43ffe3aad7999950bc7d85bdfd2cdae8fca166b68f9de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["DEV","STG"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-15T02:58:56Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x10ea921035c0\u003e: \n    failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-4426rkkk84lt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cf6cd4fd-40ae-433a-b987-569919cfb0d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cf6cd4fd-40ae-433a-b987-569919cfb0d4\",\n      \"name\": \"cf6cd4fd-40ae-433a-b987-569919cfb0d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T04:26:27.342644977Z\",\n      \"endTime\": \"2026-10-15T04:55:33.710462341Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a3g3l9y4g0x3y3s-cordon-np-rjbnx-7swmq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-4426rkkk84lt--managed/providers/Microsoft.Compute/virtualMachines/a3g3l9y4g0x3y3s-cordon-np-rjbnx-7swmq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a3g3l9y4g0x3y3s-cordon-np-rjbnx-7swmq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-4426rkkk84lt--managed/providers/Microsoft.Compute/virtualMachines/a3g3l9y4g0x3y3s-cordon-np-rjbnx-7swmq to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cf6cd4fd-40ae-433a-b987-569919cfb0d4\n            correlation ID: 442271c1-98e4-4b63-9254-ce9229dd3320\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x10ea921035c0\u003e: \n    failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-4426rkkk84lt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cf6cd4fd-40ae-433a-b987-569919cfb0d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cf6cd4fd-40ae-433a-b987-569919cfb0d4\",\n      \"name\": \"cf6cd4fd-40ae-433a-b987-569919cfb0d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T04:26:27.342644977Z\",\n      \"endTime\": \"2026-10-15T04:55:33.710462341Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a3g3l9y4g0x3y3s-cordon-np-rjbnx-7swmq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-4426rkkk84lt--managed/providers/Microsoft.Compute/virtualMachines/a3g3l9y4g0x3y3s-cordon-np-rjbnx-7swmq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a3g3l9y4g0x3y3s-cordon-np-rjbnx-7swmq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-4426rkkk84lt--managed/providers/Microsoft.Compute/virtualMachines/a3g3l9y4g0x3y3s-cordon-np-rjbnx-7swmq to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cf6cd4fd-40ae-433a-b987-569919cfb0d4\n            correlation ID: 442271c1-98e4-4b63-9254-ce9229dd3320\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2110566063010746368","occurred_at":"2026-10-15T02:58:56Z","signature_id":"1b80fec094d2059b5db09312f19fbe3bfab110461d253cb86ddd3f032d6cd4c5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5f71ef76490634700bd21f11fad8eba65488354974b95740342022fcf2257fdb","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T23:00:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x13ac3c718210\u003e: \n    resource group \"rg-nightly-4-22-mf542htcmkph\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-22-8ktwxt\" in resourcegroup=\"rg-nightly-4-22-mf542htcmkph\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cf27134-aa80-4f7f-83e4-ed95ed8adec3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cf27134-aa80-4f7f-83e4-ed95ed8adec3\",\n      \"name\": \"2cf27134-aa80-4f7f-83e4-ed95ed8adec3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:46:31.678539487Z\",\n      \"endTime\": \"2026-10-15T00:10:39.404101975Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tes3uu6rr6ujm331pj4rtffuvukefm7 still exists (deletion dispatched at 2026-10-14T23:46:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cf27134-aa80-4f7f-83e4-ed95ed8adec3\n            correlation ID: 485192e6-34cc-499a-b81c-b88c264c35e8\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x13ac3c718210\u003e: \n    resource group \"rg-nightly-4-22-mf542htcmkph\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-22-8ktwxt\" in resourcegroup=\"rg-nightly-4-22-mf542htcmkph\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cf27134-aa80-4f7f-83e4-ed95ed8adec3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cf27134-aa80-4f7f-83e4-ed95ed8adec3\",\n      \"name\": \"2cf27134-aa80-4f7f-83e4-ed95ed8adec3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:46:31.678539487Z\",\n      \"endTime\": \"2026-10-15T00:10:39.404101975Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tes3uu6rr6ujm331pj4rtffuvukefm7 still exists (deletion dispatched at 2026-10-14T23:46:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cf27134-aa80-4f7f-83e4-ed95ed8adec3\n            correlation ID: 485192e6-34cc-499a-b81c-b88c264c35e8\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110505995661742080","occurred_at":"2026-10-14T23:00:15Z","signature_id":"5a7293830fa8b7a0efa7bdd9a4ebab7d81eb7d1225302435819c0f93f39c274b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b73a2d41744880d3e5dcca5cb2a2b41591c045d73749d6123e02539cf38c2cbe","failure_pattern":"There is no failure as the matcher passed to Eventually succeeded on its most recent iteration","search_query":"There is no failure as the matcher passed to Eventually succeeded on its most recent iteration","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T23:52:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:444]: Timed out after 90.000s.\ndelegated NS and ACME TXT should resolve consistently on all authoritative nameservers, or the certificate should already be issued\nThere is no failure as the matcher passed to Eventually succeeded on its most recent iteration\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:444]: Timed out after 90.000s.\ndelegated NS and ACME TXT should resolve consistently on all authoritative nameservers, or the certificate should already be issued\nThere is no failure as the matcher passed to Eventually succeeded on its most recent iteration"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2110519049451474944","occurred_at":"2026-10-14T23:52:07Z","signature_id":"f3300b648ee9771e5ef5c9c950ba9a982e44b9a1e600ce64cde2bc827427941d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5876218c3e485902359b438d646a10e40f4e8e1df2502e7b22dee86c93191389","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":7,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-14T23:00:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:106]: failed to verify HCP cluster \"cx-rg-hcp-cluster\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x9c519e26018\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x9c519869940\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x9c519869920\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x9c5196cebe0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x9c51a1232c0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x9c51a1232a0\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x9c519493900\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:106]: failed to verify HCP cluster \"cx-rg-hcp-cluster\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x9c519e26018\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x9c519869940\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x9c519869920\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x9c5196cebe0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x9c51a1232c0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x9c51a1232a0\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x9c519493900\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110505995661742080","occurred_at":"2026-10-14T23:00:15Z","signature_id":"7fd4ca2fd062c9e1ce85412b05f7c985a192323cff82d8394f552f4cf5f7a255","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dc9165f4bd277250a09d171d82860a118abef271e78db3241118bb591c3522ee","failure_pattern":"waiting for a presented DNS-01 challenge for *","search_query":"waiting for a presented DNS-01 challenge for *","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-15T02:00:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:444]: Timed out after 90.003s.\ndelegated NS and ACME TXT should resolve consistently on all authoritative nameservers, or the certificate should already be issued\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2ce43d7a8400\u003e: \n    waiting for a presented DNS-01 challenge for *.apps.aro.ingress-cert-le.lewk.australiaeast.aroapp-hcp.io\n    {\n        s: \"waiting for a presented DNS-01 challenge for *.apps.aro.ingress-cert-le.lewk.australiaeast.aroapp-hcp.io\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:444]: Timed out after 90.003s.\ndelegated NS and ACME TXT should resolve consistently on all authoritative nameservers, or the certificate should already be issued\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2ce43d7a8400\u003e: \n    waiting for a presented DNS-01 challenge for *.apps.aro.ingress-cert-le.lewk.australiaeast.aroapp-hcp.io\n    {\n        s: \"waiting for a presented DNS-01 challenge for *.apps.aro.ingress-cert-le.lewk.australiaeast.aroapp-hcp.io\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2110551294769893376","occurred_at":"2026-10-15T02:00:15Z","signature_id":"acce76d64f91ef77fce4feb527025abc19b4be9a536a6fb8b24c4830483780d0","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":3,"failed_runs":2,"raw_occurrences":13,"matched_occurrences":13,"runs_affected":2},"rows":[{"environment":"stg","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":5,"after_last_push_count":66,"also_in":["PROD"],"trend_counts":[5,5,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":17,"prior_last_seen_at":"2026-10-14T23:00:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2b37044679e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7d2htmbslr94/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7d2htmbslr94/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n            correlation ID: 689ca8f0-a0a7-4c06-bfb9-19cd522d1f66\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2b37044679e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7d2htmbslr94/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7d2htmbslr94/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n            correlation ID: 689ca8f0-a0a7-4c06-bfb9-19cd522d1f66\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xcb290a85560\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-2tgfxvwwqb5t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-2tgfxvwwqb5t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n            correlation ID: 84cba63c-9e34-4e29-9e46-1f9b0e11372c\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xcb290a85560\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-2tgfxvwwqb5t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-2tgfxvwwqb5t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n            correlation ID: 84cba63c-9e34-4e29-9e46-1f9b0e11372c\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x17bcec4f9980\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-g6cmppj9596x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-g6cmppj9596x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n            correlation ID: b68a38b9-43ed-47b2-b847-180d273da63f\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x17bcec4f9980\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-g6cmppj9596x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-g6cmppj9596x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n            correlation ID: b68a38b9-43ed-47b2-b847-180d273da63f\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2110505995703685120","occurred_at":"2026-10-14T23:00:15Z","signature_id":"014bc043968ee0f71cff9e2ad27de77d3e537bb5fee2145f4c8b4d421c536e26","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2110505995703685120","occurred_at":"2026-10-14T23:00:15Z","signature_id":"42624eaee0ade24d65cad41064ab539262a1b91a327576387f8e93f62559c302","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2110505995703685120","occurred_at":"2026-10-14T23:00:15Z","signature_id":"850d21b1192c0bb0b380781a7e9dba4b69a395a2a5c8c76ae4c6a09aedcb3c21","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2110505995703685120","occurred_at":"2026-10-14T23:00:15Z","signature_id":"da67c8e32a59ee1380aa2a999982679fb696ceb3bd136f379bdd4e747bead93f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2110505995703685120","occurred_at":"2026-10-14T23:00:15Z","signature_id":"e31f6b1e30f8e3a43e9710b9e8b651da16be7d8f6cf85ee08f50d19f307c7e90","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"53cdeeaf6be0a342819bcbeaa249de72112012b896dbb65176b19f6c0f1a6d61","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":3,"after_last_push_count":11,"trend_counts":[1,3,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-14T23:00:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-snb95l\" at install version 4.22.0-0.nightly-multi-2026-10-14-193612\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f800f0eba0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-snb95l: failed waiting for cluster=\"cp-ystream-upgrade-5-0-snb95l\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-9pm7qw6t4kxg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b5b73d5-ad43-4248-a379-3589ba793716\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b5b73d5-ad43-4248-a379-3589ba793716\",\n      \"name\": \"5b5b73d5-ad43-4248-a379-3589ba793716\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:28:48.207254546Z\",\n      \"endTime\": \"2026-10-14T23:57:58.000803929Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b5b73d5-ad43-4248-a379-3589ba793716\n            correlation ID: 8af90513-a16b-4576-aebf-be0202178446\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-snb95l\" at install version 4.22.0-0.nightly-multi-2026-10-14-193612\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f800f0eba0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-snb95l: failed waiting for cluster=\"cp-ystream-upgrade-5-0-snb95l\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-9pm7qw6t4kxg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b5b73d5-ad43-4248-a379-3589ba793716\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b5b73d5-ad43-4248-a379-3589ba793716\",\n      \"name\": \"5b5b73d5-ad43-4248-a379-3589ba793716\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:28:48.207254546Z\",\n      \"endTime\": \"2026-10-14T23:57:58.000803929Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b5b73d5-ad43-4248-a379-3589ba793716\n            correlation ID: 8af90513-a16b-4576-aebf-be0202178446\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-4bfb22rwmqzl/cluster-nightly-4-22-8zxbz4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x363444c03360\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-8zxbz4: failed waiting for cluster=\"cluster-nightly-4-22-8zxbz4\" in resourcegroup=\"rg-nightly-4-22-4bfb22rwmqzl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e7e5bf8-d998-4cf9-b7a2-a75ca9ab02f3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e7e5bf8-d998-4cf9-b7a2-a75ca9ab02f3\",\n      \"name\": \"4e7e5bf8-d998-4cf9-b7a2-a75ca9ab02f3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:25:15.190269818Z\",\n      \"endTime\": \"2026-10-14T23:54:17.885160466Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e7e5bf8-d998-4cf9-b7a2-a75ca9ab02f3\n            correlation ID: eee8c816-0fc0-4cfa-8c09-a91c4755ee62\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-4bfb22rwmqzl/cluster-nightly-4-22-8zxbz4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x363444c03360\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-8zxbz4: failed waiting for cluster=\"cluster-nightly-4-22-8zxbz4\" in resourcegroup=\"rg-nightly-4-22-4bfb22rwmqzl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e7e5bf8-d998-4cf9-b7a2-a75ca9ab02f3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e7e5bf8-d998-4cf9-b7a2-a75ca9ab02f3\",\n      \"name\": \"4e7e5bf8-d998-4cf9-b7a2-a75ca9ab02f3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:25:15.190269818Z\",\n      \"endTime\": \"2026-10-14T23:54:17.885160466Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e7e5bf8-d998-4cf9-b7a2-a75ca9ab02f3\n            correlation ID: eee8c816-0fc0-4cfa-8c09-a91c4755ee62\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-21-7c4xdncsltnp/cluster-nightly-4-21-7xdrt5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc38fe936560\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-7xdrt5: failed waiting for cluster=\"cluster-nightly-4-21-7xdrt5\" in resourcegroup=\"rg-nightly-4-21-7c4xdncsltnp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bc06cf13-34dc-4e88-8b6c-774417a9e32b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bc06cf13-34dc-4e88-8b6c-774417a9e32b\",\n      \"name\": \"bc06cf13-34dc-4e88-8b6c-774417a9e32b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:25:05.12965498Z\",\n      \"endTime\": \"2026-10-14T23:54:07.890200924Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bc06cf13-34dc-4e88-8b6c-774417a9e32b\n            correlation ID: 0f756d77-a36e-4289-82ab-c5ef8b73feb4\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-21-7c4xdncsltnp/cluster-nightly-4-21-7xdrt5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc38fe936560\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-7xdrt5: failed waiting for cluster=\"cluster-nightly-4-21-7xdrt5\" in resourcegroup=\"rg-nightly-4-21-7c4xdncsltnp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bc06cf13-34dc-4e88-8b6c-774417a9e32b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bc06cf13-34dc-4e88-8b6c-774417a9e32b\",\n      \"name\": \"bc06cf13-34dc-4e88-8b6c-774417a9e32b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:25:05.12965498Z\",\n      \"endTime\": \"2026-10-14T23:54:07.890200924Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bc06cf13-34dc-4e88-8b6c-774417a9e32b\n            correlation ID: 0f756d77-a36e-4289-82ab-c5ef8b73feb4\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2110505995703685120","occurred_at":"2026-10-14T23:00:15Z","signature_id":"8577b00e9686687922db9c0b7c93f999df54f9a74a9f8613e6619e948f649748","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2110505995703685120","occurred_at":"2026-10-14T23:00:15Z","signature_id":"b013a0b3dd7bd242566a0e70c55ca0aeab8e5725e59a5351c6e84d763e17b209","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2110505995703685120","occurred_at":"2026-10-14T23:00:15Z","signature_id":"c0ca08d431b0ba369c6bd342618d70bdbeca775da90445e4e977816be0b6b4d3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"34aecee469de1d884ea46e23ff6727d6f9c494870f6a79f955e34ff82db132ab","failure_pattern":"CreateOrUpdateExternalAuthAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-n6c4c4fd2ssk\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-n6c4c4fd2ssk, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T02:00:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:262]: failed to apply external auth config with prefix policy None\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x95f8488fc80\u003e: \n    failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-n6c4c4fd2ssk\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-n6c4c4fd2ssk, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-n6c4c4fd2ssk\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-n6c4c4fd2ssk, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x95f84458c30\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-n6c4c4fd2ssk\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:262]: failed to apply external auth config with prefix policy None\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x95f8488fc80\u003e: \n    failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-n6c4c4fd2ssk\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-n6c4c4fd2ssk, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-n6c4c4fd2ssk\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-n6c4c4fd2ssk, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x95f84458c30\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-n6c4c4fd2ssk\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2110189439996137472","occurred_at":"2026-10-14T02:00:55Z","signature_id":"9a01b0129009725c13101877a59ee924feac36de5d9367732d0e5b62c59478f5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"072798ffcbed0f651fed67b6aaffe5e051c9690d785fd2438357172a703df137","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T23:00:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-zxqcgtfv7nf5/cluster-nightly-5-1-2dz545 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3ec63f648620\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-2dz545: failed waiting for cluster=\"cluster-nightly-5-1-2dz545\" in resourcegroup=\"rg-nightly-5-1-zxqcgtfv7nf5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c1d8733b-0494-46c8-9bd3-0a4108ec0b5c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c1d8733b-0494-46c8-9bd3-0a4108ec0b5c\",\n      \"name\": \"c1d8733b-0494-46c8-9bd3-0a4108ec0b5c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:26:17.571795133Z\",\n      \"endTime\": \"2026-10-14T23:55:18.071165518Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, ingress-operator, cluster-image-registry-operator, hosted-cluster-config-operator, cluster-storage-operator, olm-operator, redhat-operators-catalog, community-operators-catalog, certified-operators-catalog, cluster-policy-controller, catalog-operator, packageserver, openshift-controller-manager, dns-operator, openshift-route-controller-manager, cluster-version-operator, cluster-network-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c1d8733b-0494-46c8-9bd3-0a4108ec0b5c\n            correlation ID: b45fa2ae-f53a-4948-b0fb-190aed3be0b8\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-zxqcgtfv7nf5/cluster-nightly-5-1-2dz545 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3ec63f648620\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-2dz545: failed waiting for cluster=\"cluster-nightly-5-1-2dz545\" in resourcegroup=\"rg-nightly-5-1-zxqcgtfv7nf5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c1d8733b-0494-46c8-9bd3-0a4108ec0b5c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c1d8733b-0494-46c8-9bd3-0a4108ec0b5c\",\n      \"name\": \"c1d8733b-0494-46c8-9bd3-0a4108ec0b5c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:26:17.571795133Z\",\n      \"endTime\": \"2026-10-14T23:55:18.071165518Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, ingress-operator, cluster-image-registry-operator, hosted-cluster-config-operator, cluster-storage-operator, olm-operator, redhat-operators-catalog, community-operators-catalog, certified-operators-catalog, cluster-policy-controller, catalog-operator, packageserver, openshift-controller-manager, dns-operator, openshift-route-controller-manager, cluster-version-operator, cluster-network-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c1d8733b-0494-46c8-9bd3-0a4108ec0b5c\n            correlation ID: b45fa2ae-f53a-4948-b0fb-190aed3be0b8\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2110505995703685120","occurred_at":"2026-10-14T23:00:15Z","signature_id":"4a6e7765d3b234137a6519fa0eadc2627ae32cab7f766618456d71059c53b5a2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"433ac3e41264da043a734cacbf5bc6523c2571e994c16b98d54f5b82c0cc686a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-14T23:00:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-qfmkcj\" at install version 5.0.0-0.nightly-multi-2026-10-14-152123\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33329457b0a0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-1-qfmkcj: failed waiting for cluster=\"cp-ystream-upgrade-5-1-qfmkcj\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-667r4hptzmjc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b88eb811-d667-4408-bb78-6c86786b44f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b88eb811-d667-4408-bb78-6c86786b44f0\",\n      \"name\": \"b88eb811-d667-4408-bb78-6c86786b44f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:29:34.061824976Z\",\n      \"endTime\": \"2026-10-14T23:58:37.90403656Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, ingress-operator, openshift-controller-manager, cluster-network-operator, cluster-image-registry-operator, catalog-operator, openshift-route-controller-manager, packageserver, cluster-version-operator, cluster-storage-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, hosted-cluster-config-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b88eb811-d667-4408-bb78-6c86786b44f0\n            correlation ID: 8de1d04b-8264-4a06-8468-c991e9773a81\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-qfmkcj\" at install version 5.0.0-0.nightly-multi-2026-10-14-152123\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33329457b0a0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-1-qfmkcj: failed waiting for cluster=\"cp-ystream-upgrade-5-1-qfmkcj\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-667r4hptzmjc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b88eb811-d667-4408-bb78-6c86786b44f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b88eb811-d667-4408-bb78-6c86786b44f0\",\n      \"name\": \"b88eb811-d667-4408-bb78-6c86786b44f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:29:34.061824976Z\",\n      \"endTime\": \"2026-10-14T23:58:37.90403656Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, ingress-operator, openshift-controller-manager, cluster-network-operator, cluster-image-registry-operator, catalog-operator, openshift-route-controller-manager, packageserver, cluster-version-operator, cluster-storage-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, hosted-cluster-config-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b88eb811-d667-4408-bb78-6c86786b44f0\n            correlation ID: 8de1d04b-8264-4a06-8468-c991e9773a81\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2110505995703685120","occurred_at":"2026-10-14T23:00:15Z","signature_id":"18a0ae50b6aeeacf6f4e2169a6603a154559f41ed8efa4b344067bb6d640fc63","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"756ec1486dfe53bd8f251f9e5c9464445aa9f8f5c115b686b34fa380aa5cb026","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T23:00:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-0-8qt7kcrvhtlb/cluster-nightly-5-0-6wsq4v should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30f3685a760\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-6wsq4v: failed waiting for cluster=\"cluster-nightly-5-0-6wsq4v\" in resourcegroup=\"rg-nightly-5-0-8qt7kcrvhtlb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/70ee1e74-bffb-4dec-89fb-282756ca2270\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/70ee1e74-bffb-4dec-89fb-282756ca2270\",\n      \"name\": \"70ee1e74-bffb-4dec-89fb-282756ca2270\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:26:07.782117382Z\",\n      \"endTime\": \"2026-10-14T23:55:07.903819031Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-0.nightly-multi-2026-10-14-152123 is Partial (want Completed), started 20m19s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/70ee1e74-bffb-4dec-89fb-282756ca2270\n            correlation ID: e153794f-fb7c-4e3b-9441-a5dadf7b0225\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-0-8qt7kcrvhtlb/cluster-nightly-5-0-6wsq4v should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30f3685a760\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-6wsq4v: failed waiting for cluster=\"cluster-nightly-5-0-6wsq4v\" in resourcegroup=\"rg-nightly-5-0-8qt7kcrvhtlb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/70ee1e74-bffb-4dec-89fb-282756ca2270\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/70ee1e74-bffb-4dec-89fb-282756ca2270\",\n      \"name\": \"70ee1e74-bffb-4dec-89fb-282756ca2270\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:26:07.782117382Z\",\n      \"endTime\": \"2026-10-14T23:55:07.903819031Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-0.nightly-multi-2026-10-14-152123 is Partial (want Completed), started 20m19s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/70ee1e74-bffb-4dec-89fb-282756ca2270\n            correlation ID: e153794f-fb7c-4e3b-9441-a5dadf7b0225\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2110505995703685120","occurred_at":"2026-10-14T23:00:15Z","signature_id":"cfa2d92e2d5e33e9e6d62d768b07871fceff620e2949484d11e89c2a14df07c5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"38696c55a4f3351eeef461deb7069c172bd5ca5ddee5655459be99523f263138","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["DEV","PROD"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-13..2026-10-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-14T02:00:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e2df3fce3a0\u003e: \n    failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-cxh9xv97rj9f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0195b61d-2798-45a3-a546-b2e316a8e4be\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0195b61d-2798-45a3-a546-b2e316a8e4be\",\n      \"name\": \"0195b61d-2798-45a3-a546-b2e316a8e4be\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T03:10:09.167253269Z\",\n      \"endTime\": \"2026-10-14T03:39:12.28722327Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v9u7s9d4e2s3d9f-cordon-np-j7dj6-snwxb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-cxh9xv97rj9f--managed/providers/Microsoft.Compute/virtualMachines/v9u7s9d4e2s3d9f-cordon-np-j7dj6-snwxb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v9u7s9d4e2s3d9f-cordon-np-j7dj6-snwxb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-cxh9xv97rj9f--managed/providers/Microsoft.Compute/virtualMachines/v9u7s9d4e2s3d9f-cordon-np-j7dj6-snwxb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0195b61d-2798-45a3-a546-b2e316a8e4be\n            correlation ID: 6295da04-1f90-4641-a016-0502957f2d44\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e2df3fce3a0\u003e: \n    failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-cxh9xv97rj9f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0195b61d-2798-45a3-a546-b2e316a8e4be\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0195b61d-2798-45a3-a546-b2e316a8e4be\",\n      \"name\": \"0195b61d-2798-45a3-a546-b2e316a8e4be\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T03:10:09.167253269Z\",\n      \"endTime\": \"2026-10-14T03:39:12.28722327Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v9u7s9d4e2s3d9f-cordon-np-j7dj6-snwxb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-cxh9xv97rj9f--managed/providers/Microsoft.Compute/virtualMachines/v9u7s9d4e2s3d9f-cordon-np-j7dj6-snwxb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v9u7s9d4e2s3d9f-cordon-np-j7dj6-snwxb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-cxh9xv97rj9f--managed/providers/Microsoft.Compute/virtualMachines/v9u7s9d4e2s3d9f-cordon-np-j7dj6-snwxb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0195b61d-2798-45a3-a546-b2e316a8e4be\n            correlation ID: 6295da04-1f90-4641-a016-0502957f2d44\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2110189439996137472","occurred_at":"2026-10-14T02:00:55Z","signature_id":"fb5045f1b194ee1b057ae791bec58fa7b505b98c7c93aa828d301e5d4be214e3","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}