{"meta":{"start_date":"2026-10-12","end_date":"2026-10-18","start_at":"2026-10-12T00:00:00Z","end_at":"2026-10-19T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T05:05:42Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":491,"failed_runs":364,"raw_occurrences":781,"matched_occurrences":757,"runs_affected":340},"rows":[{"environment":"dev","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":61,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":61,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,61,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE starts a control plane z-stream upgrade with an SRE pin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3cd963448a00\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-g99wvvnb2gqt\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/feb6dbe3-4518-4432-8ee1-e82bfe257e23\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/feb6dbe3-4518-4432-8ee1-e82bfe257e23\",\n      \"name\": \"feb6dbe3-4518-4432-8ee1-e82bfe257e23\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T11:18:40.122769052Z\",\n      \"endTime\": \"2026-10-16T11:47:49.816923189Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/feb6dbe3-4518-4432-8ee1-e82bfe257e23\n            correlation ID: 6e26fa23-6c8f-48c7-84e4-93c4480408ac\n            client request ID: 0de8699c-e23a-4388-aef8-31bf31c0384d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3cd963448a00\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-g99wvvnb2gqt\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/feb6dbe3-4518-4432-8ee1-e82bfe257e23\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/feb6dbe3-4518-4432-8ee1-e82bfe257e23\",\n      \"name\": \"feb6dbe3-4518-4432-8ee1-e82bfe257e23\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T11:18:40.122769052Z\",\n      \"endTime\": \"2026-10-16T11:47:49.816923189Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/feb6dbe3-4518-4432-8ee1-e82bfe257e23\n            correlation ID: 6e26fa23-6c8f-48c7-84e4-93c4480408ac\n            client request ID: 0de8699c-e23a-4388-aef8-31bf31c0384d\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:163]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x390e05a68de0\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-xqknpzfzmr5w\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d23223-112b-4b7f-ba15-3c3397a86d65\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d23223-112b-4b7f-ba15-3c3397a86d65\",\n      \"name\": \"73d23223-112b-4b7f-ba15-3c3397a86d65\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T11:18:15.514720241Z\",\n      \"endTime\": \"2026-10-16T11:47:24.729519268Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d23223-112b-4b7f-ba15-3c3397a86d65\n            correlation ID: 88facd09-aef2-4a7f-9c7e-db87f515aac5\n            client request ID: 4c75b22c-aded-487f-a952-f949adbe49c6\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:163]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x390e05a68de0\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-xqknpzfzmr5w\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d23223-112b-4b7f-ba15-3c3397a86d65\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d23223-112b-4b7f-ba15-3c3397a86d65\",\n      \"name\": \"73d23223-112b-4b7f-ba15-3c3397a86d65\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T11:18:15.514720241Z\",\n      \"endTime\": \"2026-10-16T11:47:24.729519268Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d23223-112b-4b7f-ba15-3c3397a86d65\n            correlation ID: 88facd09-aef2-4a7f-9c7e-db87f515aac5\n            client request ID: 4c75b22c-aded-487f-a952-f949adbe49c6\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-g5rdgl7zwjl6/cluster-candidate-4-20-k9b9wf should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x18f77dd205c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-k9b9wf: failed waiting for cluster=\"cluster-candidate-4-20-k9b9wf\" in resourcegroup=\"rg-candidate-4-20-g5rdgl7zwjl6\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0984890f-0a19-47d9-9a1c-275d44c6e314\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0984890f-0a19-47d9-9a1c-275d44c6e314\",\n      \"name\": \"0984890f-0a19-47d9-9a1c-275d44c6e314\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T11:18:36.959680763Z\",\n      \"endTime\": \"2026-10-16T11:47:39.816204138Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0984890f-0a19-47d9-9a1c-275d44c6e314\n            correlation ID: c1d2f06b-ddfc-4fd2-8766-74fa80998c3b\n            client request ID: ef5cac13-1a05-485d-b15a-d283225aac9d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-g5rdgl7zwjl6/cluster-candidate-4-20-k9b9wf should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x18f77dd205c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-k9b9wf: failed waiting for cluster=\"cluster-candidate-4-20-k9b9wf\" in resourcegroup=\"rg-candidate-4-20-g5rdgl7zwjl6\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0984890f-0a19-47d9-9a1c-275d44c6e314\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0984890f-0a19-47d9-9a1c-275d44c6e314\",\n      \"name\": \"0984890f-0a19-47d9-9a1c-275d44c6e314\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T11:18:36.959680763Z\",\n      \"endTime\": \"2026-10-16T11:47:39.816204138Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0984890f-0a19-47d9-9a1c-275d44c6e314\n            correlation ID: c1d2f06b-ddfc-4fd2-8766-74fa80998c3b\n            client request ID: ef5cac13-1a05-485d-b15a-d283225aac9d\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"02885cebd9f78ad3acdecfda8dfeea89bd33beff91262b016a2ea9791fecec67","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"05538f072fdbd9e2a7d3c80b025b5f0230de28b40a21437ec1e51553c1f9bd1e","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"057841c6d731f163f47fffd09f49fa90b912e57433629c7df3d0e0334e9e5e9c","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"089aef486c6063d842b930743dd8e473f8a41298562aa03e74f23d7f080684bb","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"0a4b69efc91d14544bf2507c26816357641f81a32c9c58337f77d325cca61d19","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"0accb990492557d084ad00c9a810ebe72dd8287b7bc8d9d394a79d3ec69fbd86","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"0fd93a4c34a4de4e23345b99b01e77944e8813b5762ee29651e30a253ad3a392","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"107ada83ebd0b578c73b8f47d78199af675df1d0228d381e1c3b379e802effee","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"187cdd3ccfea7e8b9a553c281602c3a4ea6ca7108b24fd37be5a95399b1285a3","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"1b5b51cd57c129f4bd09991200713b69d34af14417d439c9d03facc0f7809f38","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"1d37cf0777cda01567c7d0c86c154a99941b6bfaea816e2afa28caec0e946b5a","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"20cab90c333e74780b28476d91e59f4c5e920fd33e4db9625862d05048d14bb9","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"278afa23556e267f104e90f63791a567ee1b91c5dae6d332f1391b922d1ce932","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"2bee3baecdf764b673e8d30ec79e81ae8ad2ba4a2f7c7205567bf951a79600bc","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"2d1cfe3f27ef0904cf407ecfb62faae7ea943492cd64b78f52ceb19f586165cf","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"396888b5a0686222585c428a9bce3005511dca83925bb8f3900a64c106989f35","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"3e127dc8a5033c99ca0089d8b9177daa434c14c29e269fe237c27a7ead0ffd82","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"3f03c97f235f01d6999956f4cc53d2f97cf7d8ec8ef75b33c128cf78a34ef8b5","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"3f0e76f754f9f39ad23ff397a68c8bda34243b425a937395a42136cdfada3fde","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"41b4e4ab1b6b9f7c1850160dbccf81ec34d905cb168826f78de485a244e050f8","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"42e98ecdecbc4839cf618f859d9c4e963fd3f5fa82e5c8c1be6af5035a269d5c","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"42fa5aa1fc3c312b35f41fb9e8f7ab9f22d0742794def38b6d37010c17c73253","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"494d8441a417398603173f97d342c28db189843720bc98016b63a187b18c50ad","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"49b4d135392c1a8dfe2a53dd66eb8524c2043df0a329f9fd87409c7f19e72b70","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"4be11067a4063ad348db95dbcf80fdc2805fe4486a6b4ce644af9294d77bbbb5","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"4dcc9eff48755be1b1a70a880ea55753d6810313149150fb1254bf0ca5907e39","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"518f777d390e10ac4956e3dca8c437996930936e6b6c2b05b130888e0b602e46","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"51a306129e11393c28658e5503ee4f1ab0f2982ff766d534a0cb44e56e7afb0d","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"5cc11cee2be1a364a4a8f96d02040c08c6400f6da963a61909443a73c2e23f5a","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"5f52cde04677912060df6f64e39e81dc642c9922c6a6f3cd67da57388aaedb2e","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"627de531eb0c6e8642332de2ab9eab9bc93f70b8522fb01255fbad68a41cf741","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"63b7a609dba699a3bb0a33d635a9e5513671bbfb16836cadb91c5d2dcd8c6f19","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"6b59025aff4632ced9c54e21a7d8ba849c8f13e1bb860284550175feaf03cd58","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"701100ab256952d9a5de8b351e6a3b385a113afbfeffcd255acd7e6f24e3a347","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"721ede543ee7748f0c12fb1646a03204d96825f09741021d26520978ddc98f51","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"7942f5dbb02de53dfe9b25dd229b9629fc800c349fe2c7f397013613de82a0a9","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"7d04c13ed582cb2d55761c556286e5033b490f884607524524f1777b4ec41755","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"94a66c07cad70ee5eaeb4c1ee5f1c2f9726f80978ce10d86eaeb30122d478883","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"9647c888b9bb5db2ae4c02f3ef1f1edb946e9fd90922589e62703226b412cc0f","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"97485c8b78318018cb7e1ff269311e2c04f020e9ccc2721c72cf0b76e8b10422","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"977474b948f7cd6d6e0c39010a84ec7957861d87ffc87786f2fb4bc1065ae482","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"97b54c62c879084f09800d76b8478521b07e47adf303c9bf8aa4fb3023c81fe2","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"99c69765f43fb35fd9ac8d86d7dd4823005533fe8821d9261a6c6ec54ba28e58","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"9abb8da4d45c27b5c241d75cfb1dcf171e174ac6a95fe33a4e43b4d92dc7e7bf","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"a2f68ea0fe59a89d13063468f53fb9a82f7c10a74a1836c761d6416cb5beff47","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"a55604cd1e12d3c298c747acbe1545adf0311ab0a9c1e30c8e1feeaa1450872c","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"ae79e903fb8883e6a16fdc0341c2befe1b7b1220a63ea3653584d1992ada9cb3","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"c4d0ed7d70c4e2723c49ed37fb9159b5ca2ef302da886789f78b6a44414ea242","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"cd976b5f6960ce8d52b4b03b10926d1a311c4c83fb44987456f12b9d1d004aac","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"ce4b2abc9b3ceb68bbd021867c606e5a5cb6e1e7afd106b40ea6fb89ee7a23a6","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"cf1cbb296deea632da0aa91ba2e9058c868e017af626fd893c179b4c9f1c65c1","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"d174aa6363d992122c49f4fc21a9483030773776409c9e301f2a19c69d947b2a","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"d4eef89b2cf8249a4c0863dbed753ac1980c9cb8463f4f58576a4a7e55a4fc59","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"d5f86ee2aac4ee5b3ef45df0c2a654bb28ac6de1273c99cc3362c3b527063993","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"d654a19b4359e6f92e0522049300de4d0adadb00699b4a6c37b2ea868ce4a692","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"dcb01c2ddc87c79812f3a6b8a813ececccf2210f79f2960a1f189bc60733454e","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"df7cfa109123f6b9d56e9533cf7581cd5ae0ca66c8a0568d1106a07da3e58237","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"e272ef11785652552361f893f5a45defb3d287faaaaf2b541ca001f902cbf560","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"e47f272ea8d6238d4737d12adebe1d3669788d8b3f1a6689760a033749bf48c8","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"ec0f46269a3e3f9e6756c36a09c2cdcdf54acfc291e0b485ea33cb3efad67bc3","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"eef2dbac73ba59a5bfe3c29d485600cdcc6126efca87a76b6b356a29aecedab8","pr_number":6982,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":57,"runs_affected":57,"failed_runs":57,"run_impact_percent":11.608961303462321,"anchor_occurrences":57,"after_last_push_count":17,"trend_counts":[0,0,2,26,18,10,1,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":57,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":57}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x35a206020900\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x35a2060208e0\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x35a2051a8cd0\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x35a2060208c0\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x35a206b44300\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x35a206020900\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x35a2060208e0\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x35a2051a8cd0\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x35a2060208c0\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x35a206b44300\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x225983c28ba0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x225983c28b60\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x225983a80280\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x225983c28b20\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x22598544d170\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x225983c28ba0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x225983c28b60\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x225983a80280\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x225983c28b20\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x22598544d170\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a4a4982c560\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2a4a4982c540\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x2a4a477c0190\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x2a4a4982c520\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x2a4a49889830\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a4a4982c560\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2a4a4982c540\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x2a4a477c0190\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x2a4a4982c520\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x2a4a49889830\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7382/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111575207289294848","occurred_at":"2026-10-17T21:48:55Z","signature_id":"223637359840624c4e713b2fd6730d870d033cda522f1c6b3d937aab39c513d1","pr_number":7382,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111108963322826752","occurred_at":"2026-10-16T14:56:13Z","signature_id":"1a4ee126e99baefb3096a97ac2cd1348a9667da209b6831ca0327a37c66e723e","pr_number":7232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111090334573268992","occurred_at":"2026-10-16T13:42:12Z","signature_id":"c6a5ed957cb1161fe411a2fb10ae2652cf0af6041a491232f29a014245c182c2","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111088872313065472","occurred_at":"2026-10-16T13:36:23Z","signature_id":"fb1ea24c5e9ac0e2100d89dced160bc12a43efdc23d9fc0e7e7e9dd8e55ddc2a","pr_number":7577,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111084766907338752","occurred_at":"2026-10-16T13:20:05Z","signature_id":"f39c90cd5040db84704fbb6ee3d67b497be28152e614a615d759d414b915f724","pr_number":7582,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111079254782382080","occurred_at":"2026-10-16T12:58:10Z","signature_id":"b2293d4fa65dfff28dd996462bab11c3af45d26e4fa09b72ed06e5be8af20cbf","pr_number":7556,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7523/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111072522505031680","occurred_at":"2026-10-16T12:31:25Z","signature_id":"78897419cb3b43b7451376611892509b6c60c0d6e400e895529a13bc21831597","pr_number":7523,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111032217462378496","occurred_at":"2026-10-16T09:51:15Z","signature_id":"f7c1c17ba2f883d4fd6f04f522549121b520b9cd1aa76eeb64893db3518e6496","pr_number":7575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7564/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110960931310669824","occurred_at":"2026-10-16T05:08:00Z","signature_id":"f816a0ce46c83f26ec513b8e3bd7f4adea18fdccd58c65277a306190fa1fe08f","pr_number":7564,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110943321047175168","occurred_at":"2026-10-16T03:57:57Z","signature_id":"3f087f03f689e64c84aa13709a63b29a529c080753b6b960c4759f2524e920f5","pr_number":7132,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110939401772077056","occurred_at":"2026-10-16T03:42:27Z","signature_id":"8f88f36aa920c3c11d09edc503e4d36cde94fa4e3555afb3adfaf72dcfd404e5","pr_number":7250,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7570/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110860905989279744","occurred_at":"2026-10-15T22:30:32Z","signature_id":"7dc49856cbf97ddae1c45c85ef17ff77e22e9fa432ac90a6d122e254431ca54d","pr_number":7570,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110837905944481792","occurred_at":"2026-10-15T20:59:08Z","signature_id":"e9281c8cb589d467c8e1734e954a8b4ff42420f25944517f0414599eb996f6bd","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7382/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110810466912571392","occurred_at":"2026-10-15T19:10:04Z","signature_id":"e3a9c751bb2f30f839c233d844791d0a6b5baa31994b3a806ae9b630dad57d1d","pr_number":7382,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110787442339483648","occurred_at":"2026-10-15T17:38:37Z","signature_id":"2af4f31c817166f4aea46317dbe104162254f8d1ee8f4f8a4df652f5413bfe51","pr_number":7566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110772387883520000","occurred_at":"2026-10-15T16:38:43Z","signature_id":"44b06e709c5ece379a921e93ac8f827fcdf059af1fac1ae0b57df533ad60249d","pr_number":7560,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110765471857905664","occurred_at":"2026-10-15T16:11:19Z","signature_id":"df06f989e049ce03e0acba445a6ee94f3bb5f4da0de8eadf644d1e72db0fe169","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110750802271277056","occurred_at":"2026-10-15T15:13:01Z","signature_id":"5daa74ff25889a009cfb1cd9784e8fb9af3aed81e45f7d05969bce76d6728f4a","pr_number":7463,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110722422222098432","occurred_at":"2026-10-15T13:20:15Z","signature_id":"ad5f37fb561ba9d457bb185e2bb6ee07afaa828a6b8109a6cc04032270ef77bf","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110717569160712192","occurred_at":"2026-10-15T13:00:26Z","signature_id":"4381aac8c9701a2fe445d819289027b9058d3436f2191196ee5d155f57310242","pr_number":7480,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110652237393956864","occurred_at":"2026-10-15T08:41:22Z","signature_id":"0a88b60f490c372a793b77f6a12c9179264a6df02978e62f0a32848ccfe61c53","pr_number":6944,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7487/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110607153617178624","occurred_at":"2026-10-15T05:42:13Z","signature_id":"f41e6789e6b26c9adf70978b7436d49fb8bbf2b1fc6cc62fa3dffca6d08aa5fc","pr_number":7487,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7546/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110562220516577280","occurred_at":"2026-10-15T02:43:37Z","signature_id":"45c4f42cbc3394e8d69b5ccff8a904b2f9c2403ade9ecdfd1010b4b987308c68","pr_number":7546,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7498/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558456200040448","occurred_at":"2026-10-15T02:28:42Z","signature_id":"1949375ff2b581247bba104a1ddd7d7e7b539444467f08076c41b3a836b79e5c","pr_number":7498,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558444497932288","occurred_at":"2026-10-15T02:28:40Z","signature_id":"dfca8e66f25d1dc47b86b0bdd51891ee8106201ba523c149c3a5bcaa770a0fe5","pr_number":7492,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7499/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558435291435008","occurred_at":"2026-10-15T02:28:37Z","signature_id":"4ab7144c5c500292b3e3173f19ed52ce36626055729a5b45c564a3a540c45a66","pr_number":7499,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558373865852928","occurred_at":"2026-10-15T02:28:23Z","signature_id":"4a98ab6d29d4ac6ec2b8ed5c01e37d793c68fc2131bcd10feecd606b87143b86","pr_number":7524,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558354509139968","occurred_at":"2026-10-15T02:28:18Z","signature_id":"ceca02b107a69f0963dfed1a5c6ae5b5b588366f559e0f44fcbdb03e555d8ea5","pr_number":7509,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110534454291206144","occurred_at":"2026-10-15T00:53:20Z","signature_id":"e24fe37523dd8477f2e9248cefea3d57f206a693446feef12e700343453024ae","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7532/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110513999702396928","occurred_at":"2026-10-14T23:32:03Z","signature_id":"e61871927b40d79ab621de3d7b04db3309cb36b6da4099f014a1287bcddd206a","pr_number":7532,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110500986920374272","occurred_at":"2026-10-14T22:40:21Z","signature_id":"8b6ff30d5729c06336599627d1753d772466979c7d321edead0ce4abc023e2f2","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110500816379973632","occurred_at":"2026-10-14T22:39:40Z","signature_id":"df64f40d70cc038ea7e186e9fe3ec3ba843b2428158b72034bbd34caf4fb5367","pr_number":7342,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110483415672295424","occurred_at":"2026-10-14T21:30:23Z","signature_id":"ea54a393ba27dfd62e41cfe9e83dadd2af63caf674e37f0a01a3ab250f594c72","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110456066088112128","occurred_at":"2026-10-14T19:41:51Z","signature_id":"5724b1afa43f955853aa69aef7840da478df768f7adac5bef575c3065271b7e3","pr_number":7502,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7498/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452354565279744","occurred_at":"2026-10-14T19:27:06Z","signature_id":"7979b20855eabda20007f6470ef29f513c1d6135e9e8a5039320fb7f4ffa22f4","pr_number":7498,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452322852147200","occurred_at":"2026-10-14T19:26:58Z","signature_id":"61be387220ef87ff768c357b2113308f70f1e46f3b2c65bbda3df6db4746431d","pr_number":7497,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452284272939008","occurred_at":"2026-10-14T19:26:49Z","signature_id":"8a6c53fba1289eafddee3dc2209da4cc4da34923949b74e88f7af84c0c16efbe","pr_number":7500,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110444073914470400","occurred_at":"2026-10-14T18:54:11Z","signature_id":"26cc3f18c688e60a4525b3a85a9535f2ebbb9f5b82f993744500f76041d5e350","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7527/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110408382648160256","occurred_at":"2026-10-14T16:32:22Z","signature_id":"3407f3c6628610125e781d12dcb7b736191590fb6f84e2cb2605a7040e22e20e","pr_number":7527,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110407002151718912","occurred_at":"2026-10-14T16:26:53Z","signature_id":"03beb3e937976fe9c9cc83f1032b6fe1c93c15757c83dbb637e830643a12476e","pr_number":7524,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110389003348873216","occurred_at":"2026-10-14T15:15:22Z","signature_id":"600f850e5b0064c6e16a842f50c39fbcf04bb8615687624b565925b81e9272cd","pr_number":7500,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110371287573467136","occurred_at":"2026-10-14T14:04:58Z","signature_id":"446971e32c8d7eaa419e50808053148289030b264b2065526ee8ad44598d6bc2","pr_number":7524,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110361307805913088","occurred_at":"2026-10-14T13:25:18Z","signature_id":"f21ed846cfee8ba8329873d162dbbca110a0827914a71233b54bd9b0e887e5d1","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"00d6afff568ec9fe8320e006deebf9a2ec333eeafd0113a7207c3401059e8288","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110334902741241856","occurred_at":"2026-10-14T11:40:23Z","signature_id":"33bbace3f82b158f3b3934cc2b9a2c887834084f3072bc89bf02086c9319be95","pr_number":7500,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110334900086247424","occurred_at":"2026-10-14T11:40:22Z","signature_id":"1fd2861a2ecdecb50432c0aaea802d173b55f8803b47a00d6ba6388a6f98a3c1","pr_number":7492,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110334374363795456","occurred_at":"2026-10-14T11:38:17Z","signature_id":"3e29d39bd8590975516a2c5d3e94fd98154d3dfc49c7ae3cbae6f51d43e795ef","pr_number":7502,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110305876723109888","occurred_at":"2026-10-14T09:44:59Z","signature_id":"84b1932c2aad8c16a68cba4715462d64dc0cf19c6b12f77659cab41f2c157772","pr_number":7497,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110269833781186560","occurred_at":"2026-10-14T07:21:49Z","signature_id":"9cffe9e337d6e68d2d1815dd3533f556856d9bf85b4da15334b89c4b7a8023d2","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7518/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110268638186442752","occurred_at":"2026-10-14T07:16:28Z","signature_id":"31fcba85fdd69c41767174d3dfec81cf15e72884a0829fbd9bb13aa3cf481d6d","pr_number":7518,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110267140211740672","occurred_at":"2026-10-14T07:11:07Z","signature_id":"39b08081edae55a01b00ee6d278d4fe5b4565d42932b3eb0793613aa32d74b95","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110267078714855424","occurred_at":"2026-10-14T07:10:50Z","signature_id":"84ff6b9fa357efbe37c1955bfcfcc7167434d092376b6767fb221f43e1ee5f81","pr_number":7519,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110235250528358400","occurred_at":"2026-10-14T05:04:24Z","signature_id":"388fa5f884dc9873a3e0ac5e7934c037c1b2aca0708794c9638a9509d5d20e5f","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110213970672488448","occurred_at":"2026-10-14T03:39:51Z","signature_id":"c3f223ea41e95da8d49b0cc04384dc9d6bba824c5592c8a1fc0a741ad927ca6d","pr_number":7504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110168566228586496","occurred_at":"2026-10-14T00:39:25Z","signature_id":"d34b77d805ee032e385bd7e9ec93a0f17a9c99b3eb797f5536963db648176fbe","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110129974156464128","occurred_at":"2026-10-13T22:06:04Z","signature_id":"694e2d897de6010d0383c8b74dec7d08484cd381d7c5e3824ebcf322dba221fb","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110112563529256960","occurred_at":"2026-10-13T20:56:53Z","signature_id":"efb9c37e965d24b97d7e8ab613bd6ca3432ad4802836e8dcd3a2df165e5003e5","pr_number":7469,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21","occurrences":56,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.814663951120163,"anchor_occurrences":56,"after_last_push_count":1,"trend_counts":[0,42,0,9,5,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":5,"prior_last_seen_at":"2026-09-29T01:37:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE starts a control plane z-stream upgrade with an SRE pin","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:114]: failed to create cluster \"sre-version-pin-k97lng\" at 4.21.36\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1533100288e0\u003e: \n    failed to create HCP cluster sre-version-pin-k97lng: failed waiting for cluster=\"sre-version-pin-k97lng\" in resourcegroup=\"admin-api-version-pin-srtm7jx8t2zb\" to finish creating: GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b84e5174-a51f-4ccb-81ff-7fe8387b9ff7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b84e5174-a51f-4ccb-81ff-7fe8387b9ff7\",\n      \"name\": \"b84e5174-a51f-4ccb-81ff-7fe8387b9ff7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T09:40:00.953675448Z\",\n      \"endTime\": \"2026-10-15T10:09:07.839497723Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b84e5174-a51f-4ccb-81ff-7fe8387b9ff7\n            correlation ID: ee96e857-47ed-4d45-8af0-6c84276e732e\n            client request ID: 54710fb8-fb6a-4b6b-b646-27d3dc3d8364\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:114]: failed to create cluster \"sre-version-pin-k97lng\" at 4.21.36\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1533100288e0\u003e: \n    failed to create HCP cluster sre-version-pin-k97lng: failed waiting for cluster=\"sre-version-pin-k97lng\" in resourcegroup=\"admin-api-version-pin-srtm7jx8t2zb\" to finish creating: GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b84e5174-a51f-4ccb-81ff-7fe8387b9ff7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b84e5174-a51f-4ccb-81ff-7fe8387b9ff7\",\n      \"name\": \"b84e5174-a51f-4ccb-81ff-7fe8387b9ff7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T09:40:00.953675448Z\",\n      \"endTime\": \"2026-10-15T10:09:07.839497723Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b84e5174-a51f-4ccb-81ff-7fe8387b9ff7\n            correlation ID: ee96e857-47ed-4d45-8af0-6c84276e732e\n            client request ID: 54710fb8-fb6a-4b6b-b646-27d3dc3d8364\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-fzffs7\" with version 4.21.36 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2f680e6086a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-fzffs7: failed waiting for cluster=\"cluster-zstream-4-21-fzffs7\" in resourcegroup=\"rg-zstream-upgrade-4-21-fk78wfgpv6f4\" to finish creating: GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f992823-ace9-4bbf-93da-fc0443bfd8c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f992823-ace9-4bbf-93da-fc0443bfd8c7\",\n      \"name\": \"4f992823-ace9-4bbf-93da-fc0443bfd8c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T09:40:08.845848533Z\",\n      \"endTime\": \"2026-10-15T10:09:17.834288486Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f992823-ace9-4bbf-93da-fc0443bfd8c7\n            correlation ID: 3da74456-40a1-495e-a2ee-ae1aa1e064b2\n            client request ID: 063711c3-7c8b-4258-ac53-1eeaec215430\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-fzffs7\" with version 4.21.36 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2f680e6086a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-fzffs7: failed waiting for cluster=\"cluster-zstream-4-21-fzffs7\" in resourcegroup=\"rg-zstream-upgrade-4-21-fk78wfgpv6f4\" to finish creating: GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f992823-ace9-4bbf-93da-fc0443bfd8c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f992823-ace9-4bbf-93da-fc0443bfd8c7\",\n      \"name\": \"4f992823-ace9-4bbf-93da-fc0443bfd8c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T09:40:08.845848533Z\",\n      \"endTime\": \"2026-10-15T10:09:17.834288486Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f992823-ace9-4bbf-93da-fc0443bfd8c7\n            correlation ID: 3da74456-40a1-495e-a2ee-ae1aa1e064b2\n            client request ID: 063711c3-7c8b-4258-ac53-1eeaec215430\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-7xw44w\" with version 4.20.41 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1e014d2781c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-7xw44w: failed waiting for cluster=\"cluster-zstream-4-20-7xw44w\" in resourcegroup=\"rg-zstream-upgrade-4-20-m9dzg9b4mkpj\" to finish creating: GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/40a40396-7b07-4032-84dc-85483287fa77\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/40a40396-7b07-4032-84dc-85483287fa77\",\n      \"name\": \"40a40396-7b07-4032-84dc-85483287fa77\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T09:39:48.143600215Z\",\n      \"endTime\": \"2026-10-15T10:08:57.782039441Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/40a40396-7b07-4032-84dc-85483287fa77\n            correlation ID: c1f76f8b-0244-4f84-9b33-78a9ec6f02fc\n            client request ID: 435434cd-57a6-4a90-8709-e50f7498118a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-7xw44w\" with version 4.20.41 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1e014d2781c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-7xw44w: failed waiting for cluster=\"cluster-zstream-4-20-7xw44w\" in resourcegroup=\"rg-zstream-upgrade-4-20-m9dzg9b4mkpj\" to finish creating: GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/40a40396-7b07-4032-84dc-85483287fa77\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/40a40396-7b07-4032-84dc-85483287fa77\",\n      \"name\": \"40a40396-7b07-4032-84dc-85483287fa77\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T09:39:48.143600215Z\",\n      \"endTime\": \"2026-10-15T10:08:57.782039441Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9473792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/40a40396-7b07-4032-84dc-85483287fa77\n            correlation ID: c1f76f8b-0244-4f84-9b33-78a9ec6f02fc\n            client request ID: 435434cd-57a6-4a90-8709-e50f7498118a\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110655550189473792","occurred_at":"2026-10-15T08:54:31Z","signature_id":"6ea6dc28ca29ce388eaea5eaff7424f8fb9c7372f41ab9247b712fce4570770d","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110655550189473792","occurred_at":"2026-10-15T08:54:31Z","signature_id":"854df219ee64fc312863156844f77c0cdc80fcf9a0c6ed5c8f78c43658697afb","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110655550189473792","occurred_at":"2026-10-15T08:54:31Z","signature_id":"9448d19885e88c1e5c9239f3c806cb52207d101f22ecefde29b28902b4bc9e3a","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110655550189473792","occurred_at":"2026-10-15T08:54:31Z","signature_id":"abe72ba7916e50e38f7592f8c814dfd42449491278dc023cfeadfefabf04e325","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110655550189473792","occurred_at":"2026-10-15T08:54:31Z","signature_id":"c12e3545f9668e73b16d0e434c163d09d165e972516a2fa11c89fb3d3b8ef626","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110436010226618368","occurred_at":"2026-10-14T18:22:06Z","signature_id":"143def6409b35b1c7d757163265b805bf1b1688f374686fad28f34887ebc3acc","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110436010226618368","occurred_at":"2026-10-14T18:22:06Z","signature_id":"24aeb68202ddb418951a2358babdeb1022f31ae3504b6e8ebf13c0be83fcf229","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110436010226618368","occurred_at":"2026-10-14T18:22:06Z","signature_id":"6f9cf8d30ba68d9f16ced08b0ca5a22bded54c97f83f0a19c451ec450b4450ad","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110436010226618368","occurred_at":"2026-10-14T18:22:06Z","signature_id":"9de95f427e1e691bee870aa47fe10797350a1d239b34784b0123ece1c0d37502","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110436010226618368","occurred_at":"2026-10-14T18:22:06Z","signature_id":"ac1f00c956f42dd7a153687ebb08193db10a4521feb9fe5150ebbc130419e571","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"179b78cf112d379960e54d9d4b5b459a8f1cc6bfa3bbb31725a79978e9184c64","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"4b731517316ce3a5808ec2ba5ecb34ecc20952fffb5e19de022a57c410f4e2f4","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"9ed4cd1b6f3c34dbaf9c5e754cc24b690545748ca53215644722020cca18bdae","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"e7075774c82d3677e833d8f6c5fca557dda0e4dfa59ef31282500a86e593da0b","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"055f9a9b6c244ec70223014863e02389b9a479621a6d56cd84fc1a590883325e","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"06d984b5bd61e16ead3a82aa893ca35dc9577f417c04093909481769e3f573b8","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"0a7b025a664cd937a27c1586ab0853b5c1ef3166363dadc1edebbd26f45f0f59","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"160071fa1d66581da10fab1353ab09d80ff5734975070cab9434ca0e1322ff49","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"1ec3f7dc73b39cc393755b265af0c51797463b27d90c21e0a2834ab16544dbf0","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"40e270b1250c572d95ad297676751fd03a52912e32aa17a878d7239f4fb1933c","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"434f1f50bc3fec5a453eb5fc6ce070a5ad6cf40f73119f0ef622c86873aa56ea","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"44ded8bb1a8d8a2ac19d80e715a411a4b8efc7b7cc9f887538780c1e34b93ab9","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"47dea9fa6d473ca88a0d6daa3a8a61ce6d04ebe38764f2f66d84f10a437111c6","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"4d85fc8c6214af114002c843ebdc86a94cf3126e2e1355356d3ebd73640aa90f","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"51071ac504e6050a1fd884cd82764171a1d38ab7411e9ae858f536e87567f055","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"586593e2fff2e69d645fc42dff24f41310743ceba1deccc3b657c96749c0928b","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"5bc4f6a77655afb07a6100dcfe611731a929b63429a3f3272baa0c1801222753","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"5c7543311511f19fcd39fd25b976d88fc18cf2a06f89b8cd4da4fdcb65f57861","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"5f344ec756d50afef29819a6882bec64ddbd8b7b94b117d33badf95b4228e5c9","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"61789678564be0aad54c9098dc019c63c0a07abf6db9351123669661958b90b1","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"6b095616bded2736b44fd15719c1d4b68c8ddb854dc8491ee2f30e52888a0be7","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"6e1b7db0d8aeab353439c24ad4514730ee14c5e2353bca21858fe1f9675242cc","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"7c6abafe393255f1739fdb387bd94e187d718daaaf8baf7ec69854b4ddf141b2","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"7cb425f2d7951985395b91345c374be4c0dca4d08831fcfb07f410b4baec6807","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"8181f041b308c072f1833b4eb56517c581e5c8e4e9cd0a8ea0e6808fccc7a57f","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"844afb351725b6475ddd327ef78fbf5da706650b28665d4a6e378e60144b8e25","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"85acf0e937ffe0027b15bd44eca4c7c42194560ce1678ed387a13da6d7aadcb6","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"945c8b8ba229efce78c82c55117d9808b09ecb3e3f5b4cd8cda0f81bc283543f","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"97fe8f81d50ab0c04cc31936087a1cb6ccf4abd2ff0897e188700ec58b85890f","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"9b8bdf9e25347ed8bbd86e38e47f8af0335efec4efa7ea5222d2c6561d3fadfb","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"9f01d929e13b36781c18a8d7f12e2b1fcb9913e0a3fcb25d7d5ff885c820a357","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"9f58aa2dc15f7e7336a495843cc0f699c3e5b143f75bf07f3842358d658bcdca","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"a0f1bf26985a99e6539af709b7c6cfaeed3b89c57f24089e9a344841e1966c77","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"a312b9a0ce9a609eef608f7ee363a1a717dc275fe1d0ca99a6b39591f5f7c10b","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"b2b126547a0bcacd7f8d6fadea91d4a58f4981406c524e6014bd0b7da6c92d4d","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"b7d3cff9604baa7adb7447f8feddb9b4e1f30933ea4005d1bf550006ccbe8684","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"b83badcc40496e121c06a8423ede3d7aeada8dd0924a4287300c02e0fa12d3e6","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"c7ae04b38766210a8108f94bb04d957dfc5bca36ff0aa8754d485ec2530ed6a3","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"d1805d4cf77820ef4f29d07237f0247a441e0228a2174ebcae5ceacb2de61aa9","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"d859241facd880cf6291230bc4ecdb749dce2456ddc8c4a192721ae181952d05","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"da295e3f891ebd53e53b834c83d5f2d2d8c7990e91838251fdcf5881a2a1f90a","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"dbf86b79040a8aaf8dd94a9133ea7c86d12cf15986aa135f1c6a41485545be9c","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"e642f26ea0a98605c1da7d415345036ed06aed0f4668bc7504474fbfa4f815a5","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"e76fe38f94cc0c24f57a2b14af3523d40d7b5c15cd0e3fefac98dc8b85b99bed","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"ed62ccd125102b89fe1e6cd6a42a65c596b410194665770b4c1ecb63e1ef3a7a","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"ed6cd23384ef866a46342ced82dc3a93106a5447a643b0459d4e1595765f54a8","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":46,"runs_affected":46,"failed_runs":46,"run_impact_percent":9.368635437881874,"anchor_occurrences":46,"after_last_push_count":82,"trend_counts":[2,8,9,12,11,6,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":243,"prior_last_seen_at":"2026-10-11T17:11:29Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":46}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-16T17:47:40Z\n  Ended:       2026-10-16T17:58:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1762432-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-16T17:00:31Z\n  Ended:       2026-10-16T17:16:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5648768-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-16T17:00:31Z\n  Ended:       2026-10-16T17:14:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5648768-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-16T14:54:18Z\n  Ended:       2026-10-16T15:01:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3268992-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111133911231762432","occurred_at":"2026-10-16T16:35:11Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":7588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7587/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111124374575648768","occurred_at":"2026-10-16T15:57:28Z","signature_id":"dfb324349c1f93796329f7da2f6b37edd36593be9c02584b4eddb23c7aa504b8","pr_number":7587,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111090334573268992","occurred_at":"2026-10-16T13:42:12Z","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"19b6ab86b15a2714ad1a515f08f8394ebbf9aba03e8c697d567a805fe6d0e44f","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111034368125308928","occurred_at":"2026-10-16T09:59:49Z","signature_id":"b398e3b4b60ce0bfe92ea3c19658030c4ec5dee4a44f8a5f5c5664c590af5113","pr_number":7387,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7561/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110949609818820608","occurred_at":"2026-10-16T04:23:01Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7561,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110787442339483648","occurred_at":"2026-10-15T17:38:37Z","signature_id":"ec53f16855846d8bcc63c32e61dda8320f5f786dcafa5b90527c9de5094abd44","pr_number":7566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7382/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110720043451944960","occurred_at":"2026-10-15T13:10:48Z","signature_id":"ec53f16855846d8bcc63c32e61dda8320f5f786dcafa5b90527c9de5094abd44","pr_number":7382,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110717161084293120","occurred_at":"2026-10-15T12:59:19Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7557,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7555/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110697898764144640","occurred_at":"2026-10-15T11:42:48Z","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","pr_number":7555,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7552/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110682140487192576","occurred_at":"2026-10-15T10:40:11Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7552,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7489/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110673619796365312","occurred_at":"2026-10-15T10:06:19Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7489,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7532/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110649423133413376","occurred_at":"2026-10-15T08:30:11Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7532,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7530/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110626518202847232","occurred_at":"2026-10-15T06:59:10Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7530,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110588894188670976","occurred_at":"2026-10-15T04:29:39Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7498/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558456200040448","occurred_at":"2026-10-15T02:28:42Z","signature_id":"b398e3b4b60ce0bfe92ea3c19658030c4ec5dee4a44f8a5f5c5664c590af5113","pr_number":7498,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558351342440448","occurred_at":"2026-10-15T02:28:17Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110505977575903232","occurred_at":"2026-10-14T23:00:10Z","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7526/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110504372508037120","occurred_at":"2026-10-14T22:53:42Z","signature_id":"1055a81696a8c72ee71de5548a9db39421bfe798741063f2b4483fcbb49ce2e8","pr_number":7526,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110500986920374272","occurred_at":"2026-10-14T22:40:21Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7528/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110483351910486016","occurred_at":"2026-10-14T21:30:15Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7528,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110483335657558016","occurred_at":"2026-10-14T21:30:12Z","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","pr_number":7524,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6866/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110463708953055232","occurred_at":"2026-10-14T20:12:13Z","signature_id":"ec53f16855846d8bcc63c32e61dda8320f5f786dcafa5b90527c9de5094abd44","pr_number":6866,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110456066088112128","occurred_at":"2026-10-14T19:41:51Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7502,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110412598238253056","occurred_at":"2026-10-14T16:49:06Z","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","pr_number":7529,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7428/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110376503379234816","occurred_at":"2026-10-14T14:25:41Z","signature_id":"b398e3b4b60ce0bfe92ea3c19658030c4ec5dee4a44f8a5f5c5664c590af5113","pr_number":7428,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7522/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110350863582105600","occurred_at":"2026-10-14T12:43:47Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7522,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110303886584582144","occurred_at":"2026-10-14T09:37:08Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7383,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110265029910597632","occurred_at":"2026-10-14T07:02:43Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7516,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110080447856775168","occurred_at":"2026-10-13T17:40:39Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7501/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110034896461238272","occurred_at":"2026-10-13T15:47:07Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7501,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7499/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031817837907968","occurred_at":"2026-10-13T15:36:01Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7499,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110028695728033792","occurred_at":"2026-10-13T15:23:37Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7469,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110028457319600128","occurred_at":"2026-10-13T15:22:41Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109977127079120896","occurred_at":"2026-10-13T11:58:43Z","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","pr_number":7463,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7484/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109972734892052480","occurred_at":"2026-10-13T11:41:15Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7484,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7491/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109971633794977792","occurred_at":"2026-10-13T11:36:53Z","signature_id":"d06483cd9806ffb19df33df26852e61ee3baea8bf5cda6e46715e7ad95a9157e","pr_number":7491,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109729919306043392","occurred_at":"2026-10-12T19:36:16Z","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109629712874606592","occurred_at":"2026-10-12T12:58:13Z","signature_id":"ec53f16855846d8bcc63c32e61dda8320f5f786dcafa5b90527c9de5094abd44","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109622919087788032","occurred_at":"2026-10-12T12:31:13Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109612109212946432","occurred_at":"2026-10-12T11:48:16Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7467,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109600459105767424","occurred_at":"2026-10-12T11:01:58Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7468,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109555123024826368","occurred_at":"2026-10-12T08:00:48Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":6746,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":37,"runs_affected":37,"failed_runs":37,"run_impact_percent":7.535641547861507,"anchor_occurrences":37,"after_last_push_count":39,"trend_counts":[4,5,6,4,10,9,1,2],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":199,"prior_last_seen_at":"2026-10-11T23:29:37Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":37}],"full_error_samples":["alert fired 38 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-tmw4b\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-tmw4b has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-4fdv6\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-4fdv6 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-5vc4f\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-5vc4f has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-dfx48\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-dfx48 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-tmkfh\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-tmkfh has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-ldq7g\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-ldq7g has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-4prhm\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-4prhm has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-mtvd8\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-mtvd8 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-w9lfx\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-w9lfx has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-m6kzd\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-m6kzd has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-gf4mg\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-gf4mg has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-jtg79\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-jtg79 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-gbc9d\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-gbc9d has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-vnwdn\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-vnwdn has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-cvdcq\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-cvdcq has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-jp4zq\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-jp4zq has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-lcvwv\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-lcvwv has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-nllhh\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-nllhh has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-x8lvp\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-x8lvp has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-2cp2r\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-2cp2r has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-bnghx\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-bnghx has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-spjrm\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-spjrm has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-7gwqf\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-7gwqf has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-wzhd4\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-wzhd4 has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-z67pp\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-z67pp has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-8rjbk\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-8rjbk has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-89fp5\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-89fp5 has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-dfhsg\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-dfhsg has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-ndv7w\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-ndv7w has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-x65xf\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-x65xf has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-9nhcz\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-9nhcz has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-57rg5\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-57rg5 has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-vqfkk\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-vqfkk has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-mvbzv\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-mvbzv has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-xqzzm\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-xqzzm has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-pzz7r\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-pzz7r has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-4ht2v\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-4ht2v has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-m2n57\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-m2n57 has been in a non-ready state for longer than 5 minutes.","alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-18T02:22:20Z\n  Ended:       2026-10-18T02:28:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5039232-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgtj1aja7eu3kumpui0v9v0hrn0ctrn-n2t2k5k1d2q7b4m\", pod=\"openshift-apiserver-74747bbdf5-l6sx2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgtj1aja7eu3kumpui0v9v0hrn0ctrn-n2t2k5k1d2q7b4m/openshift-apiserver-74747bbdf5-l6sx2 has been in a non-ready state for longer than 5 minutes.","alert fired 32 time(s) (25 unknown, 7 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-17T09:53:56Z\n  Ended:       2026-10-17T10:04:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgf9u7qesjgmttof4tbj175kd9c3v6a-j2l3z7n5l1o3m5w\", pod=\"network-node-identity-5479889bd6-wm9n5\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgf9u7qesjgmttof4tbj175kd9c3v6a-j2l3z7n5l1o3m5w/network-node-identity-5479889bd6-wm9n5 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-17T09:53:56Z\n  Ended:       2026-10-17T10:04:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgf9u7qesjgmttof4tbj175kd9c3v6a-j2l3z7n5l1o3m5w\", pod=\"ovnkube-control-plane-6f598f876b-4lwmh\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgf9u7qesjgmttof4tbj175kd9c3v6a-j2l3z7n5l1o3m5w/ovnkube-control-plane-6f598f876b-4lwmh has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-17T09:53:57Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfa0b8h25qc9a5i20tff4l0fg95ohb-x1q7f4j4x4u2t1g\", pod=\"network-node-identity-5d7f9f545c-9fj88\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfa0b8h25qc9a5i20tff4l0fg95ohb-x1q7f4j4x4u2t1g/network-node-identity-5d7f9f545c-9fj88 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-10-17T09:56:00Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfa3i9d0dtsn3mqgm2m0jvbj0gost6-c8m7l6f8p4l9x4f\", pod=\"ovnkube-control-plane-6776f795c7-pnvf4\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfa3i9d0dtsn3mqgm2m0jvbj0gost6-c8m7l6f8p4l9x4f/ovnkube-control-plane-6776f795c7-pnvf4 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-10-17T09:56:00Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfa3i9d0dtsn3mqgm2m0jvbj0gost6-c8m7l6f8p4l9x4f\", pod=\"network-node-identity-6484b49989-tpf9j\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfa3i9d0dtsn3mqgm2m0jvbj0gost6-c8m7l6f8p4l9x4f/network-node-identity-6484b49989-tpf9j has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-10-17T09:56:01Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacm1jgb7lcrs9ivlei7ps8sq33bk-w6k4h3j3b8i6v3d\", pod=\"network-node-identity-77ddc9bb4c-pbwr7\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacm1jgb7lcrs9ivlei7ps8sq33bk-w6k4h3j3b8i6v3d/network-node-identity-77ddc9bb4c-pbwr7 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-10-17T09:56:01Z\n  Ended:       2026-10-17T10:04:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfa5d10e98e3mrvr8mdtggsfec8ur7-z9i7c7u7w1a3k9w\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfa5d10e98e3mrvr8mdtggsfec8ur7-z9i7c7u7w1a3k9w/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-10-17T09:56:01Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacm1jgb7lcrs9ivlei7ps8sq33bk-w6k4h3j3b8i6v3d\", pod=\"ovnkube-control-plane-6bd64ff8dc-l5xp2\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacm1jgb7lcrs9ivlei7ps8sq33bk-w6k4h3j3b8i6v3d/ovnkube-control-plane-6bd64ff8dc-l5xp2 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-10-17T09:58:01Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacmm0dlphe6vtc50mkmtd3jb2p8q-f2o6b4o3v3g6y5w\", pod=\"kube-apiserver-9875495b7-tqsz4\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacmm0dlphe6vtc50mkmtd3jb2p8q-f2o6b4o3v3g6y5w/kube-apiserver-9875495b7-tqsz4 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-10-17T09:58:01Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacmm0dlphe6vtc50mkmtd3jb2p8q-f2o6b4o3v3g6y5w\", pod=\"kube-apiserver-9875495b7-222sx\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacmm0dlphe6vtc50mkmtd3jb2p8q-f2o6b4o3v3g6y5w/kube-apiserver-9875495b7-222sx has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-10-17T09:58:01Z\n  Ended:       2026-10-17T10:04:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfadvi8jv413ort809d0n4kq5kno22-s2q4c0i7f2c0r2x\", pod=\"network-node-identity-5d8fccd895-ptmvr\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfadvi8jv413ort809d0n4kq5kno22-s2q4c0i7f2c0r2x/network-node-identity-5d8fccd895-ptmvr has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-10-17T09:58:01Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacmauure3rehr0ll3kf67lkonqn4-l7q9g3b3j1r5c8t\", pod=\"network-node-identity-86d7bc77f4-zzxmb\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacmauure3rehr0ll3kf67lkonqn4-l7q9g3b3j1r5c8t/network-node-identity-86d7bc77f4-zzxmb has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-10-17T09:58:01Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacmm0dlphe6vtc50mkmtd3jb2p8q-f2o6b4o3v3g6y5w\", pod=\"kube-apiserver-9875495b7-scjqk\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacmm0dlphe6vtc50mkmtd3jb2p8q-f2o6b4o3v3g6y5w/kube-apiserver-9875495b7-scjqk has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacoq552p37e3lj875ff1g01eve2p-j5s8a2u8s3o2v6k\", pod=\"kube-apiserver-5844c8889f-cbz2x\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacoq552p37e3lj875ff1g01eve2p-j5s8a2u8s3o2v6k/kube-apiserver-5844c8889f-cbz2x has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacoq552p37e3lj875ff1g01eve2p-j5s8a2u8s3o2v6k\", pod=\"kube-apiserver-5844c8889f-d6fvz\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacoq552p37e3lj875ff1g01eve2p-j5s8a2u8s3o2v6k/kube-apiserver-5844c8889f-d6fvz has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:05:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfa3iukddj4rbptbdr63ko16fjd5a0-private-kas\", pod=\"kube-apiserver-8544f679b7-zl64m\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfa3iukddj4rbptbdr63ko16fjd5a0-private-kas/kube-apiserver-8544f679b7-zl64m has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfa70s9eurd03qajc81h11klkgrmcp-q6y7b0q2i4d5g1m\", pod=\"network-node-identity-6cd587db9f-t4z8h\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfa70s9eurd03qajc81h11klkgrmcp-q6y7b0q2i4d5g1m/network-node-identity-6cd587db9f-t4z8h has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfa3iukddj4rbptbdr63ko16fjd5a0-private-kas\", pod=\"kube-apiserver-8544f679b7-5fcvk\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfa3iukddj4rbptbdr63ko16fjd5a0-private-kas/kube-apiserver-8544f679b7-5fcvk has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:04:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfadpgoo9rvetrl85b24o6ppv4j0fo-o3i3s1q6c0r8x5c\", pod=\"network-node-identity-5d97d9b5cb-4k88v\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfadpgoo9rvetrl85b24o6ppv4j0fo-o3i3s1q6c0r8x5c/network-node-identity-5d97d9b5cb-4k88v has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:04:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacn5q7o0r4p4o2djer1ifh59adg8-ea-listq25ppj\", pod=\"network-node-identity-74f67fb4bf-ngwc7\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacn5q7o0r4p4o2djer1ifh59adg8-ea-listq25ppj/network-node-identity-74f67fb4bf-ngwc7 has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacoq552p37e3lj875ff1g01eve2p-j5s8a2u8s3o2v6k\", pod=\"kube-apiserver-5844c8889f-b42qs\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacoq552p37e3lj875ff1g01eve2p-j5s8a2u8s3o2v6k/kube-apiserver-5844c8889f-b42qs has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:04:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfapf0dhnoj0q238ct23l9vk8nc5at-k7b2q6t5f3g5p6n\", pod=\"network-node-identity-fb54b4d4d-v8wgh\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfapf0dhnoj0q238ct23l9vk8nc5at-k7b2q6t5f3g5p6n/network-node-identity-fb54b4d4d-v8wgh has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:04:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfadpgoo9rvetrl85b24o6ppv4j0fo-o3i3s1q6c0r8x5c\", pod=\"ovnkube-control-plane-6895bcbdc6-c9kvm\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfadpgoo9rvetrl85b24o6ppv4j0fo-o3i3s1q6c0r8x5c/ovnkube-control-plane-6895bcbdc6-c9kvm has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n  State:       Resolved\n  Started:     2026-10-17T09:58:59Z\n  Ended:       2026-10-17T10:05:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfa3iukddj4rbptbdr63ko16fjd5a0-private-kas\", pod=\"kube-apiserver-8544f679b7-q8rkz\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfa3iukddj4rbptbdr63ko16fjd5a0-private-kas/kube-apiserver-8544f679b7-q8rkz has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n  State:       Resolved\n  Started:     2026-10-17T09:59:55Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfa1d4f46ij8t9ghuvhfoa11l51bep-basic-cluster\", pod=\"network-node-identity-5774b9669-l5ggc\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfa1d4f46ij8t9ghuvhfoa11l51bep-basic-cluster/network-node-identity-5774b9669-l5ggc has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111927716537176064","occurred_at":"2026-10-18T21:09:39Z","signature_id":"e4b3d941e23ccd903b4c4ef9302dc754f91d92aac5112d970448006082f1a4cc","pr_number":7387,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7382/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111625789635039232","occurred_at":"2026-10-18T01:09:54Z","signature_id":"0db548d313b61e121d3be7e10a5b288fe32452c78f9c7de6b86df38a44955827","pr_number":7382,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111378042730319872","occurred_at":"2026-10-17T08:45:27Z","signature_id":"89c30c74c5d96251bd10b1bd76400fc32e088893bb54798f40173fdb211c4066","pr_number":7250,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7483/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111186249774534656","occurred_at":"2026-10-16T20:03:20Z","signature_id":"9e8e933175b0a2604bba19bae3f44ffb21292acf78887aa2195f8690115a15e3","pr_number":7483,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111133911231762432","occurred_at":"2026-10-16T16:35:11Z","signature_id":"3c51f31c3e12b890ebd28cd56bf13c145f5b755eda048613b9928abf2a5adabe","pr_number":7588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111119736791109632","occurred_at":"2026-10-16T15:39:02Z","signature_id":"5d37298322872c4c9c58f6b51822a5ad1b91a1e040cee73204cb2fb09a29f82b","pr_number":7573,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110961362036330496","occurred_at":"2026-10-16T05:09:43Z","signature_id":"78f5fc43246695b2dd39eca1a26a0ab5a62d3c615d9af627f94a79b10d68dc72","pr_number":6746,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110943321047175168","occurred_at":"2026-10-16T03:57:57Z","signature_id":"478d3a30f078ef1e264b281b0279bc5e3d4879af8a4361b3ba264c28c23360e4","pr_number":7132,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110896106962423808","occurred_at":"2026-10-16T00:50:25Z","signature_id":"847391a6f039d89e83174f4d45c5d987309f37b3c9a63e6f0f0cdd1d71ddfba6","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110895015453528064","occurred_at":"2026-10-16T00:46:04Z","signature_id":"c6522e5d94df9f7aede7ecf7a33352ae735ba2dd77d94a580cfe6f7e5bb7305c","pr_number":7471,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110894017456640000","occurred_at":"2026-10-16T00:42:06Z","signature_id":"48ac6e834ed843ec8689357e4df1c77bc51934e184727a31bf74980e2baec415","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110892726588280832","occurred_at":"2026-10-16T00:36:59Z","signature_id":"f98f21b954d35d911f699ca50bd748cba1e83339b2039f6a9126e37d1ffd7622","pr_number":7467,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7301/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110850824425967616","occurred_at":"2026-10-15T21:50:28Z","signature_id":"e700b5e28a3a57cced01c80db13b7e620639c6ae72c8ef703974cdc21d80da5c","pr_number":7301,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110822602502049792","occurred_at":"2026-10-15T19:58:20Z","signature_id":"bd069d09e58d7d95f34cdf0c374750a348e4ebf68a354ffffce03fdc37ff1eb7","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7382/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110810466912571392","occurred_at":"2026-10-15T19:10:04Z","signature_id":"fa816ee2e31223907588f9946191f5640e175ffb0ff2d3b20c314896e073acfa","pr_number":7382,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7382/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110765010153115648","occurred_at":"2026-10-15T16:09:28Z","signature_id":"ca1007ab2e98e7b07704b974b1ad91d1a5a9870ae56f540d8f0e9f4a3553351c","pr_number":7382,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110722422222098432","occurred_at":"2026-10-15T13:20:15Z","signature_id":"45db5ec120bd9da9e4674d514d8a16e9a1927d4c6b12777c820c40cf1fd662fb","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7301/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110706006915813376","occurred_at":"2026-10-15T12:15:01Z","signature_id":"c4ab6dd6e9ff4b055bd50286f866b8c099eaee5e8c1327b99fdfe97fdec6f82f","pr_number":7301,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7522/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110675476174344192","occurred_at":"2026-10-15T10:13:41Z","signature_id":"7f5a0519a65728b6ba24ff28920852871272184ab582fcfb3344b1c4844e533a","pr_number":7522,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110665048610312192","occurred_at":"2026-10-15T09:32:14Z","signature_id":"88c23eb952ff7ea7302cc4e17a29ebe6e436e42c7d53802ebeb86ccf9a8cf3e6","pr_number":7467,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7527/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110636208349515776","occurred_at":"2026-10-15T07:37:37Z","signature_id":"b1c5a5567c683302ccc0c8a0964c5ad65eb24311f0f39df9f284d5d26f849708","pr_number":7527,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110588894188670976","occurred_at":"2026-10-15T04:29:39Z","signature_id":"1b16c2939746e07666536f5967563ec30a93299a662662a970a43f500af9b120","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110447856593670144","occurred_at":"2026-10-14T19:09:11Z","signature_id":"92d0db446a1dc21a2aad36ec9d8dada9e5f4b84c78fc10253e30a3cae65b1b1c","pr_number":7529,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7522/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110350863582105600","occurred_at":"2026-10-14T12:43:47Z","signature_id":"8e24ab0f93da5f4c9e06cf1e6c9a5dad864de5524f0ff38944de964d0d11c1d0","pr_number":7522,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110251822873579520","occurred_at":"2026-10-14T06:10:15Z","signature_id":"0737293351a530d5699a84b32dfec7c3c5268b0328193be453aec2ef79c7ae95","pr_number":7504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110199338117369856","occurred_at":"2026-10-14T02:41:42Z","signature_id":"17761d523d7af03109fc74ba2a7283ce86a08f29c2f5ee6925a34b270860af93","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110115702512816128","occurred_at":"2026-10-13T21:09:22Z","signature_id":"5df3df51c4f1a79ff78be7277fb6033fa17feb7d65772f7a0900bc893660bf39","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110099886413189120","occurred_at":"2026-10-13T20:04:06Z","signature_id":"ab4dcac27ea15b642051449befb4d2d369e9937091c6b496b59f54afa24883cd","pr_number":7480,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110075452461682688","occurred_at":"2026-10-13T18:01:34Z","signature_id":"920e9198dd57a8245eaf4811aaa20836cb9e3378296216a9590cd685a85e0bbf","pr_number":7469,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"cd835111d6723302e38161708c21c4a3215a58c651e4b372392909f248560622","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109977127079120896","occurred_at":"2026-10-13T11:58:43Z","signature_id":"b0c665dc5593bf939be3f561a3b7d6c563fa192fba4413e100dc7bc2db835e54","pr_number":7463,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109952299815145472","occurred_at":"2026-10-13T10:20:03Z","signature_id":"153f2db7e7ffaab6160a3f35da43a89c7da37790c513bc78fc32bb4dd3b5f592","pr_number":6944,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109729919306043392","occurred_at":"2026-10-12T19:36:16Z","signature_id":"4e6814427594ada40105d0c851f37c68ce6084d56587c92bb86a0a5911644b78","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109660256270290944","occurred_at":"2026-10-12T14:59:35Z","signature_id":"36ccc2be84055641dbb6036217141bc4c104ca4b184f2aa2e64ec66e411c6dbe","pr_number":7373,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"4885c3029699261783a90f8a5a1cf25063711e5d2be18ed2e9aec6344e30e22d","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"eb4f11f09029df76c1cd0c8884283989b59eee33dde5e273ada7dc186fddbab8","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7430/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109519598771507200","occurred_at":"2026-10-12T05:40:39Z","signature_id":"6f8e2d109f354a776a1acb2c7fa60149b6a8d7d0329aa7b37c469a8320c9fe0f","pr_number":7430,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":35,"runs_affected":35,"failed_runs":35,"run_impact_percent":7.128309572301426,"anchor_occurrences":35,"after_last_push_count":27,"trend_counts":[1,5,13,8,9,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-10-05"],"prior_runs_affected":61,"prior_last_seen_at":"2026-10-11T21:12:37Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":35}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T21:26:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j4315776-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.179:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4315776/providers/microsoft.monitor/accounts/services-j4315776\", namespace=\"klusterlet-2tfeeaom86b60erngar03ad51ehpgqtm\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-h4l8s\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2tfeeaom86b60erngar03ad51ehpgqtm on cluster ci01-j4315776-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T18:05:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j7905664-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7905664/providers/microsoft.monitor/accounts/services-j7905664\", namespace=\"klusterlet-2tfbjf51so3bj30gni3l73dohrjlrc07\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mz4bk\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2tfbjf51so3bj30gni3l73dohrjlrc07 on cluster ci01-j7905664-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T18:00:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j3950080-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.234:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3950080/providers/microsoft.monitor/accounts/services-j3950080\", namespace=\"klusterlet-2tfbf4h0d9nqbkbie85eg76q9fu1hc63\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-zrvv8\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2tfbf4h0d9nqbkbie85eg76q9fu1hc63 on cluster ci01-j3950080-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110813601194315776","occurred_at":"2026-10-15T19:22:32Z","signature_id":"c5b6f7bd77490cb4798684c5ce13459c25402b8086273a3ab23a1d0282a3f7ff","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110765471857905664","occurred_at":"2026-10-15T16:11:19Z","signature_id":"86e590f066cdb01cb3cab994fc303b2f6d2b0ef508c11d3f5327344b481ba083","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110765041643950080","occurred_at":"2026-10-15T16:09:36Z","signature_id":"3883d3564a56914c40bd2690548dc3ee9cbb4e90ee3f2ea0c6d469cb10682d57","pr_number":7373,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7489/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110733173536067584","occurred_at":"2026-10-15T14:02:58Z","signature_id":"1bda6daa416bbfa280d931353283e650a720e33291bdde4870b53d128ddc841e","pr_number":7489,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7527/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110702760532905984","occurred_at":"2026-10-15T12:01:55Z","signature_id":"d376e2d9a3830a51f306bf475add960c8ed1f411786387a6087323eb34510ff7","pr_number":7527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7532/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110649423133413376","occurred_at":"2026-10-15T08:30:11Z","signature_id":"266d9b9ad446b09ddd2b3e559fbed88f1176aa40b805689e9ca1e1ca6f7b4b28","pr_number":7532,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7530/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110626518202847232","occurred_at":"2026-10-15T06:59:10Z","signature_id":"458aa86e7797c40e0ab03abce032560dcdb9988bd7bf8dcebedd58d4e8bbf23d","pr_number":7530,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110588894188670976","occurred_at":"2026-10-15T04:29:39Z","signature_id":"1d84b1a7cfc29b2eaa012fb4fc792bc3e3ed7989684d13143ab9e7705252ea88","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558354509139968","occurred_at":"2026-10-15T02:28:18Z","signature_id":"6368b0c6fa0c1de7f3094acf13f7071ec55cea65182b85f7bbf296ca29822fb2","pr_number":7509,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7540/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110508385492275200","occurred_at":"2026-10-14T23:09:45Z","signature_id":"3c96117c5ddc1261bfec64ae3985a8f8906d46b2a771385f7492378201af02c6","pr_number":7540,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7538/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110474243606056960","occurred_at":"2026-10-14T20:54:04Z","signature_id":"d0ae0ee40a93e9522e8aa3aad5e2e8d13a87da569f13779e1b12906aa8c9b53c","pr_number":7538,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452322852147200","occurred_at":"2026-10-14T19:26:58Z","signature_id":"76b6f718f1f07d60590d0948a4fdfeda4a1da54d0c8b8a64b4f169c6a0d37c98","pr_number":7497,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110437162552922112","occurred_at":"2026-10-14T18:26:44Z","signature_id":"423d326e2b1972473d10fe07604ba7cb4534248683d2ec096f5ae5959abcce41","pr_number":7524,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7527/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110408382648160256","occurred_at":"2026-10-14T16:32:22Z","signature_id":"fcf669af0e8c7da39eaec1b22f419c476a115f903e8455b3aff82defb226e02c","pr_number":7527,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7525/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110338041116626944","occurred_at":"2026-10-14T11:52:51Z","signature_id":"a5775537c5d4a02a7df57596f722ad13cd1cea1e62fcb6e5097655900ef1ec2b","pr_number":7525,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7499/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110304557253791744","occurred_at":"2026-10-14T09:39:48Z","signature_id":"f560bfc31c39b650b322b862f5a16de94b12597c676328f7859afb2592dc53de","pr_number":7499,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110272422715330560","occurred_at":"2026-10-14T07:30:12Z","signature_id":"9f5a880ad682e61c10851eb2f1014abff11271943a08f54b06f8a578999bf9d7","pr_number":7441,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110075452461682688","occurred_at":"2026-10-13T18:01:34Z","signature_id":"dea3871c0f87e0576e988742c681e43a182dd667db806dd19107b71b0897c1c0","pr_number":7469,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110043835399671808","occurred_at":"2026-10-13T16:23:47Z","signature_id":"75b1290532c2ce606ea20bb999b66cb1911da401025d28a68dcef6b1a4ebb765","pr_number":7504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110039512951296000","occurred_at":"2026-10-13T16:06:36Z","signature_id":"c8d1bc3849a91c7150052cbd8fd01e8bacd6acfee15dec3a5b9f1ee2a8d69a1f","pr_number":7488,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110039423042195456","occurred_at":"2026-10-13T16:05:54Z","signature_id":"e0aece362630a726f3a415cc6e772ac06ec61590d1f6668bb57f6d646bc54566","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7489/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110035013809475584","occurred_at":"2026-10-13T15:48:44Z","signature_id":"dc0f6ac01acdec662cd08a68e54dfaa04653844d8ecf2ea1d8d22da7ec3a472c","pr_number":7489,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7499/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031817837907968","occurred_at":"2026-10-13T15:36:01Z","signature_id":"545a94509aea8d174b7c626867e4c3ee62e7a7063a644a5bd20cd3227ea583e7","pr_number":7499,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"d7646a570dc56c8aaf2ff17561f2018e209352457d4d221d6f98298c58002476","pr_number":7496,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109994317174018048","occurred_at":"2026-10-13T13:07:01Z","signature_id":"84707bded3f4fb605d70b06f7b06b6aaa5f660163675ed498765d69734732aa9","pr_number":7488,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109985456534851584","occurred_at":"2026-10-13T12:31:49Z","signature_id":"e0d8a21444c75df32d1ee0ca0a5977901884526027e6eba129d7da65137483ea","pr_number":7441,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109977127079120896","occurred_at":"2026-10-13T11:58:43Z","signature_id":"45a81790b69f5a13e41c3f6d58353b78d39c6df6f209d0a628107acd5871dc3b","pr_number":7463,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7484/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109972734892052480","occurred_at":"2026-10-13T11:41:15Z","signature_id":"c0807ee7633cb3b74a1bec10eeb1a189d4d2920a14c2e55e84330c489057c130","pr_number":7484,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109920002722238464","occurred_at":"2026-10-13T08:11:43Z","signature_id":"a19d81f743cded8d0f7583e6aed1c76e28428956863bd4963ceab29f6d4d2f68","pr_number":7488,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109797931459547136","occurred_at":"2026-10-13T00:06:39Z","signature_id":"4a20c7552b13dbe39fee7488dec78de85e3480b9fcb8c75ca75d30c3902dc95f","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109754180636774400","occurred_at":"2026-10-12T21:12:48Z","signature_id":"54a7b1283931df6f61be82ca63d92bf372c766203c87971d1d688d38deb1c668","pr_number":7373,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109743222543618048","occurred_at":"2026-10-12T20:29:16Z","signature_id":"a67e3b1ce6606aadaaf58795784360bda0ae262c0ccd8be147cd4aa4d8cfa336","pr_number":7457,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109566058561015808","occurred_at":"2026-10-12T08:45:16Z","signature_id":"16da3c90424d3e20001b8e7c2d884bfc967a23a24761b0cf7cc40ac186996cdd","pr_number":7219,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7458/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109528983438626816","occurred_at":"2026-10-12T06:17:57Z","signature_id":"9a0460e320537545b7df7ed4ea07d8af2e7eacb2bec27a23e38614a9b14396d9","pr_number":7458,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109528755037802496","occurred_at":"2026-10-12T06:17:02Z","signature_id":"7f1c703708a0c73e567c797b757f2be0ee02cc20a38341bda264d7aaee221d0e","pr_number":7424,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":33,"runs_affected":33,"failed_runs":33,"run_impact_percent":6.720977596741344,"anchor_occurrences":33,"after_last_push_count":31,"trend_counts":[0,7,6,11,9,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":108,"prior_last_seen_at":"2026-10-09T22:55:44Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":33}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T21:16:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2571392-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2571392/providers/microsoft.monitor/accounts/hcps-j2571392\", namespace=\"ocm-arohcpci01-2tfehdfsiuk7rspp81ggshks0ldi46jk-d0c0o9z7l8b4h1m\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mdkdv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tfehdfsiuk7rspp81ggshks0ldi46jk-d0c0o9z7l8b4h1m/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T21:16:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2571392-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2571392/providers/microsoft.monitor/accounts/hcps-j2571392\", namespace=\"ocm-arohcpci01-2tfehdfsiuk7rspp81ggshks0ldi46jk-d0c0o9z7l8b4h1m\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mdkdv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tfehdfsiuk7rspp81ggshks0ldi46jk-d0c0o9z7l8b4h1m/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T19:40:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9990016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9990016/providers/microsoft.monitor/accounts/hcps-j9990016\", namespace=\"ocm-arohcpci01-2tfcsk6itjfjpcjhjvmmra3bj97fh52v-d7p2j6h4b2a7d8x\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-p222m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tfcsk6itjfjpcjhjvmmra3bj97fh52v-d7p2j6h4b2a7d8x/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T19:41:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9990016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9990016/providers/microsoft.monitor/accounts/hcps-j9990016\", namespace=\"ocm-arohcpci01-2tfcsk6itjfjpcjhjvmmra3bj97fh52v-d7p2j6h4b2a7d8x\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-p222m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tfcsk6itjfjpcjhjvmmra3bj97fh52v-d7p2j6h4b2a7d8x/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T18:44:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3520000-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.223:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3520000/providers/microsoft.monitor/accounts/hcps-j3520000\", namespace=\"ocm-arohcpci01-2tfbvc1dj03nufhiu0qumf4o5mmkbqp1-w6h1y1s8u1u3w6q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5k8tw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tfbvc1dj03nufhiu0qumf4o5mmkbqp1-w6h1y1s8u1u3w6q/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T18:44:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3520000-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.223:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3520000/providers/microsoft.monitor/accounts/hcps-j3520000\", namespace=\"ocm-arohcpci01-2tfbvc1dj03nufhiu0qumf4o5mmkbqp1-w6h1y1s8u1u3w6q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5k8tw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tfbvc1dj03nufhiu0qumf4o5mmkbqp1-w6h1y1s8u1u3w6q/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T18:44:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3520000-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.223:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3520000/providers/microsoft.monitor/accounts/hcps-j3520000\", namespace=\"ocm-arohcpci01-2tfbvc1dj03nufhiu0qumf4o5mmkbqp1-w6h1y1s8u1u3w6q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5k8tw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tfbvc1dj03nufhiu0qumf4o5mmkbqp1-w6h1y1s8u1u3w6q/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T18:44:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3520000-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.223:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3520000/providers/microsoft.monitor/accounts/hcps-j3520000\", namespace=\"ocm-arohcpci01-2tfbvc1dj03nufhiu0qumf4o5mmkbqp1-w6h1y1s8u1u3w6q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5k8tw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tfbvc1dj03nufhiu0qumf4o5mmkbqp1-w6h1y1s8u1u3w6q/cluster-api has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7382/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110810466912571392","occurred_at":"2026-10-15T19:10:04Z","signature_id":"e08cdb1a917621a7bc7e9ac3bae32917eab851f365fbf48b5aa11ef705a76d57","pr_number":7382,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7567/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110787652859990016","occurred_at":"2026-10-15T17:39:27Z","signature_id":"8d97f29c786c2805ed8d65fe144dbbc6a3667d59fbd9f36b72a95308d1825536","pr_number":7567,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110772387883520000","occurred_at":"2026-10-15T16:38:43Z","signature_id":"77dabc807b924d36a952127239116e3220596574e1888419cab733dd80962c6f","pr_number":7560,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7301/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110767880390840320","occurred_at":"2026-10-15T16:20:40Z","signature_id":"265f33185e8558313afb5cda094931f59292ab2438e4433d313712def08611c8","pr_number":7301,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110758849320325120","occurred_at":"2026-10-15T15:45:00Z","signature_id":"56a6494daa6aa4d1037d2c4190fd601ebbd5c6f38508657a1e61d5b33b8f3e58","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7522/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110675476174344192","occurred_at":"2026-10-15T10:13:41Z","signature_id":"e125e5c457310d34d0ba517886a0d9634c9aa3448c638c0dc6fdcbfb2e822f18","pr_number":7522,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7530/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110626518202847232","occurred_at":"2026-10-15T06:59:10Z","signature_id":"e17149d8136845bc27a292141df6a27ea8ab9816a537c4ab27a5590d07876e48","pr_number":7530,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110607466327707648","occurred_at":"2026-10-15T05:43:27Z","signature_id":"63ad5d8eb64723b20ad5d3cef316adc16a92e54d0efbf2073890cf7cec3f0852","pr_number":7509,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558444497932288","occurred_at":"2026-10-15T02:28:40Z","signature_id":"6ecefab153f490fd537d46ce93ebe675f7ec3355f24754aee618349d423b68c2","pr_number":7492,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110494690280214528","occurred_at":"2026-10-14T22:15:16Z","signature_id":"136d96d86a4c895fcd3c1d444abd7575070d82b11f64ab40d965df68d6fdaaf5","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110445968326397952","occurred_at":"2026-10-14T19:01:26Z","signature_id":"f40a39c9ca10509006eb38c52071e0f722fbe1b343f8d8790c565126c0289f61","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110437162552922112","occurred_at":"2026-10-14T18:26:44Z","signature_id":"cab2c47a06c7ff0ab3e837d0af7e5dadc7a44e0f4414274b4500f2b8eaa58ae9","pr_number":7524,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7301/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110412681776205824","occurred_at":"2026-10-14T16:49:27Z","signature_id":"a8e82832880b9ffa15cd92fbe758dc94bf292f90b84ee2a2b3b8eb7fbe84ceab","pr_number":7301,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7428/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110376503379234816","occurred_at":"2026-10-14T14:25:41Z","signature_id":"ae392ace2ed0c8022d2929768cf5177dca0ebce5bf93ae7f0d81e46d65074045","pr_number":7428,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"55034fbd6d3c5e94b22e5c438df96ac5629b046d02f4f8f6a566e208fb622951","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110334900086247424","occurred_at":"2026-10-14T11:40:22Z","signature_id":"55c67bf5d39db25b9b8b07942180b2686cfe0e47355a56931e067d63af27c315","pr_number":7492,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110291728471691264","occurred_at":"2026-10-14T08:48:48Z","signature_id":"9ba21becf373fa33475cb896a527b1cbf47fc159b36b23de4d6dc7754df75409","pr_number":7500,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110269833781186560","occurred_at":"2026-10-14T07:21:49Z","signature_id":"84eef7d5861896e04804ef44919b3f5111909b2b4c789a45fce7492b63e3a407","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110265029910597632","occurred_at":"2026-10-14T07:02:43Z","signature_id":"8c2aff0d3632c81340ce3a2c0cb498d1dab4f667e7346d1f4af0133ec58e4f5f","pr_number":7516,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110221944149250048","occurred_at":"2026-10-14T04:11:31Z","signature_id":"7756294fda68fe98e6d3f7d273b6b789955b562628b9575f65001f0516fb17ea","pr_number":6712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110131584773394432","occurred_at":"2026-10-13T22:12:28Z","signature_id":"81e6aedf0da4f27efde8cb67872b6903101fe3182111cf67d2dfdc5646317ecb","pr_number":7509,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110098887174787072","occurred_at":"2026-10-13T19:57:04Z","signature_id":"4587179fd233fc65eb9fdb22388faacf6ec23475c73d8a87452ef9bb1dc63ec6","pr_number":7503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"1adc25e471ad62d434570973bdec6d11ffd2186cddd280c054d7dedcef620819","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110027575299411968","occurred_at":"2026-10-13T15:15:38Z","signature_id":"683b9a267c4b1dc9692126fc62980df123b055e51ab1e62c31bcb909366380a0","pr_number":7485,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7491/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109971633794977792","occurred_at":"2026-10-13T11:36:53Z","signature_id":"d802fbeb68fcabadbc0eb4bc2365b8c44020b5b0b251941d4bface05433d26f5","pr_number":7491,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109804957828911104","occurred_at":"2026-10-13T00:34:34Z","signature_id":"2782cf0dfc0a0c20b3659d17b6c3a4c2d2b8a0833ca169e4ccba3462287f86c1","pr_number":7485,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109743222543618048","occurred_at":"2026-10-12T20:29:16Z","signature_id":"5cdd689b990f511feccbe071c3661b0d836cd6b71dcc489745470445ffd67711","pr_number":7457,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109740763322519552","occurred_at":"2026-10-12T20:19:29Z","signature_id":"1f8e213972bab57fbe586fb0bcc701f04d1d69dc2413a26b541db4737b3a7545","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"ace1503ea85ab8f2b4df98ae24ce5d1ca17bf7dd773d4068dfe89225c5ed9594","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"18f5e818c6420e89e3d88cbc04dfa007473088c4fb38a457467fa320874574d7","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109600459105767424","occurred_at":"2026-10-12T11:01:58Z","signature_id":"a980a63c43707ea1bb98420a0dc8810e1d7f957a882e906bd2844388f2b4bbc5","pr_number":7468,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","signature_id":"c2c5cbd8dc969313ed2d41b08a27fef289acf32d3365fe9e6a45d7383b454d11","pr_number":7463,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109528755037802496","occurred_at":"2026-10-12T06:17:02Z","signature_id":"36fc6cf9b7b57226fae3978e4f0d34c96bf304895ef33b2f585810930ebddbca","pr_number":7424,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-282j8gjps6m5/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":23,"runs_affected":23,"failed_runs":23,"run_impact_percent":4.684317718940937,"anchor_occurrences":23,"after_last_push_count":22,"trend_counts":[0,5,6,7,4,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":50,"prior_last_seen_at":"2026-10-09T20:50:18Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":23}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-9jbnpswbqhnc/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-9jbnpswbqhnc/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-kzwmzc2svmh2/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-kzwmzc2svmh2/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-fs95n68z9x97/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-fs95n68z9x97/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"0040558a56ca2b78035a7b68cf46efa81a3757a14f72084b294057ecc8885c49","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110728988165935104","occurred_at":"2026-10-15T13:46:20Z","signature_id":"9d8904b7fad3f06b7bd3bdd1d298979c15544ebe9033cbceb49f6b58611b359c","pr_number":6823,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110648898644086784","occurred_at":"2026-10-15T08:28:03Z","signature_id":"f3ca80ae0f7cab734a0af5391e570d2bbb6e0fe34253c2d5db0721cbff616d8a","pr_number":7509,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7544/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110554897882550272","occurred_at":"2026-10-15T02:14:34Z","signature_id":"c3f5ffc66171aee04e4ea896b6fd7ad1f5c40c428f2cc854d4cacf8b7ba6cad1","pr_number":7544,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110543177340424192","occurred_at":"2026-10-15T01:28:00Z","signature_id":"8c8792c48ba5bf46de2e19623688298335ee9162e2cbfd1034ac375c4ecb3c4c","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7544/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110520946510008320","occurred_at":"2026-10-14T23:59:39Z","signature_id":"dbdad5bcd6883b547539aa2a7128d9d6dac9b47d512abf71a69bd30baf31f61b","pr_number":7544,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7540/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110508385492275200","occurred_at":"2026-10-14T23:09:45Z","signature_id":"92f3748303a061cc6184e933650c30db9f661fd5374936ed7875565522f2b06d","pr_number":7540,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110490547654234112","occurred_at":"2026-10-14T21:58:50Z","signature_id":"5051886dbfc3c36339c7c0eb02b10fb512049eb5a0ac03511948fd813a8b24fa","pr_number":7492,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452284272939008","occurred_at":"2026-10-14T19:26:49Z","signature_id":"36fb74339a311618119525c91dbbd8ddfaca88e17b00be4e06ba05c2588ce610","pr_number":7500,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110407002151718912","occurred_at":"2026-10-14T16:26:53Z","signature_id":"add2ded45d4e7d255ace8de8f5215174d5761f231ca7fb8a87f2550584249b6b","pr_number":7524,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110213970672488448","occurred_at":"2026-10-14T03:39:51Z","signature_id":"fc73da3f64df28e28a5f3d03ec006882753ad077ad7fd997dd91714a48b160a9","pr_number":7504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110191032388816896","occurred_at":"2026-10-14T02:08:42Z","signature_id":"aba6bac3ecf6fccefb4d5d45429681cbe92d317984bf96d8fe7f780c1e7bca9a","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110137118025060352","occurred_at":"2026-10-13T22:34:27Z","signature_id":"94d1e13589455fbe3faae0d56a4527d0080bf018f5cc0df414cf6c40b010f2da","pr_number":6712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110123884639424512","occurred_at":"2026-10-13T21:41:52Z","signature_id":"983126b83f0ae7565666b720c4dde22c9930fd7261ca4bcc8961d600d3ad5a6a","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","signature_id":"be104b66a02409cc4adfb532056ddabf4b40d0214823d8003994a6056e0fd4a4","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"afb4e7bf5663c4264337c9d1e677c0444a5f93e3e544da615e860a878027556d","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110029189787684864","occurred_at":"2026-10-13T15:25:35Z","signature_id":"0af58b2f76d489b90f234310d2a3ac8b142182a0239f813bb77188e32f9dc462","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7491/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109971633794977792","occurred_at":"2026-10-13T11:36:53Z","signature_id":"c2743ca915206fd985839d671b5618ccb4332bb960f8c549fdcf65d90ddee3d2","pr_number":7491,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"6ab7f7a7d7318820073a94e892ad474901c481d146387431db8c950325a1baa6","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","signature_id":"9578272302c49c5a63e06e79a95c961c925c637a881235d46f047f8f61f08363","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"895d6dc730935dbb7c49cd2e90968b291d7469692edac24ed546ec1e8097382f","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109603048882966528","occurred_at":"2026-10-12T11:12:15Z","signature_id":"bbe6ff38e69c56b0da8c72190329d55c3f58aff3ba835f793e6ee32876e960e6","pr_number":7373,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109599905499582464","occurred_at":"2026-10-12T10:59:45Z","signature_id":"01ff5d20cf309ae0ced3e300e16a31a96dd5d737827ad14c8718b21686301352","pr_number":7457,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5hsxwfczjwp5\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":21,"runs_affected":21,"failed_runs":21,"run_impact_percent":4.276985743380855,"anchor_occurrences":21,"after_last_push_count":10,"trend_counts":[0,3,7,5,4,2,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":131,"prior_last_seen_at":"2026-10-11T12:00:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":14},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:163]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3a1e5c39e140\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-lsp7k9q4q2pz\" to finish creating: GET https://rp.j1762432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eeecd748-7cd9-428c-bdad-6172bba0e52f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eeecd748-7cd9-428c-bdad-6172bba0e52f\",\n      \"name\": \"eeecd748-7cd9-428c-bdad-6172bba0e52f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T17:39:40.172421101Z\",\n      \"endTime\": \"2026-10-16T17:58:48.49629868Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-pb4xk-8sfrz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-lsp7k9q4q2pz--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-pb4xk-8sfrz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-pb4xk-8sfrz: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-lsp7k9q4q2pz--managed/ingress-cert-le-np-1-pb4xk-8sfrz-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-lsp7k9q4q2pz--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-pb4xk-8sfrz-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3871b527-fb1e-4123-b38b-3cb2445a9fc8' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1762432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eeecd748-7cd9-428c-bdad-6172bba0e52f\n            correlation ID: 3ee22f63-ca3a-4588-b9c5-ce97876471df\n            client request ID: a644f8be-e878-4c8d-99e9-321877a22983\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:163]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3a1e5c39e140\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-lsp7k9q4q2pz\" to finish creating: GET https://rp.j1762432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eeecd748-7cd9-428c-bdad-6172bba0e52f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eeecd748-7cd9-428c-bdad-6172bba0e52f\",\n      \"name\": \"eeecd748-7cd9-428c-bdad-6172bba0e52f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T17:39:40.172421101Z\",\n      \"endTime\": \"2026-10-16T17:58:48.49629868Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-pb4xk-8sfrz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-lsp7k9q4q2pz--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-pb4xk-8sfrz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-pb4xk-8sfrz: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-lsp7k9q4q2pz--managed/ingress-cert-le-np-1-pb4xk-8sfrz-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-lsp7k9q4q2pz--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-pb4xk-8sfrz-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3871b527-fb1e-4123-b38b-3cb2445a9fc8' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1762432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eeecd748-7cd9-428c-bdad-6172bba0e52f\n            correlation ID: 3ee22f63-ca3a-4588-b9c5-ce97876471df\n            client request ID: a644f8be-e878-4c8d-99e9-321877a22983","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3d49e0c9b600\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-cqn2ccl8wpmh\" to finish creating: GET https://rp.j3090304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8\",\n      \"name\": \"57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T11:01:48.232534133Z\",\n      \"endTime\": \"2026-10-16T11:20:52.453260342Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j0j2o7l0q7m7m7f-np-delete-main-4ql8g-j89nf: NodeDoesNotExist: Node does not exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j0j2o7l0q7m7m7f-np-delete-main-4ql8g-j89nf: NotReady: * Deleting: Machine deletion in progress, stage: WaitingForInfrastructureDeletion\\n* HealthCheckSucceeded: Health check failed: Node failed to report startup in 10m0s\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3090304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8\n            correlation ID: f387d1ed-05da-402f-8ada-b632f0d02aae\n            client request ID: 36e27b8d-f92f-4579-9e74-48bd79aecb10\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3d49e0c9b600\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-cqn2ccl8wpmh\" to finish creating: GET https://rp.j3090304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8\",\n      \"name\": \"57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T11:01:48.232534133Z\",\n      \"endTime\": \"2026-10-16T11:20:52.453260342Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j0j2o7l0q7m7m7f-np-delete-main-4ql8g-j89nf: NodeDoesNotExist: Node does not exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j0j2o7l0q7m7m7f-np-delete-main-4ql8g-j89nf: NotReady: * Deleting: Machine deletion in progress, stage: WaitingForInfrastructureDeletion\\n* HealthCheckSucceeded: Health check failed: Node failed to report startup in 10m0s\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3090304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8\n            correlation ID: f387d1ed-05da-402f-8ada-b632f0d02aae\n            client request ID: 36e27b8d-f92f-4579-9e74-48bd79aecb10","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x31dfe7cead80\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-w4sdcc2f9b98\" to finish creating: GET https://rp.j9466368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1a9a190b-4bfc-40cb-9dd3-61f444b3d45a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1a9a190b-4bfc-40cb-9dd3-61f444b3d45a\",\n      \"name\": \"1a9a190b-4bfc-40cb-9dd3-61f444b3d45a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T11:58:54.751404091Z\",\n      \"endTime\": \"2026-10-15T12:28:03.314684153Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v0x6h3c4b5g9z6x-np-delete-main-cv9ln-97hs2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-w4sdcc2f9b98--managed/providers/Microsoft.Compute/virtualMachines/v0x6h3c4b5g9z6x-np-delete-main-cv9ln-97hs2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v0x6h3c4b5g9z6x-np-delete-main-cv9ln-97hs2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-w4sdcc2f9b98--managed/providers/Microsoft.Compute/virtualMachines/v0x6h3c4b5g9z6x-np-delete-main-cv9ln-97hs2 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9466368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1a9a190b-4bfc-40cb-9dd3-61f444b3d45a\n            correlation ID: a5182798-ac67-4625-a997-1317fca7e220\n            client request ID: d5f9fdbc-cb7e-4972-bee6-661a76843384\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x31dfe7cead80\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-w4sdcc2f9b98\" to finish creating: GET https://rp.j9466368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1a9a190b-4bfc-40cb-9dd3-61f444b3d45a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1a9a190b-4bfc-40cb-9dd3-61f444b3d45a\",\n      \"name\": \"1a9a190b-4bfc-40cb-9dd3-61f444b3d45a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T11:58:54.751404091Z\",\n      \"endTime\": \"2026-10-15T12:28:03.314684153Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v0x6h3c4b5g9z6x-np-delete-main-cv9ln-97hs2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-w4sdcc2f9b98--managed/providers/Microsoft.Compute/virtualMachines/v0x6h3c4b5g9z6x-np-delete-main-cv9ln-97hs2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v0x6h3c4b5g9z6x-np-delete-main-cv9ln-97hs2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-w4sdcc2f9b98--managed/providers/Microsoft.Compute/virtualMachines/v0x6h3c4b5g9z6x-np-delete-main-cv9ln-97hs2 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9466368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1a9a190b-4bfc-40cb-9dd3-61f444b3d45a\n            correlation ID: a5182798-ac67-4625-a997-1317fca7e220\n            client request ID: d5f9fdbc-cb7e-4972-bee6-661a76843384"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111133911231762432","occurred_at":"2026-10-16T16:35:11Z","signature_id":"b689e896267fb659ab2567cecd3a930517666e0cf7750b5517f7c69837dd5c9d","pr_number":7588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111032815893090304","occurred_at":"2026-10-16T09:53:39Z","signature_id":"0954bd1b1bf2ab0efaa5338771ec18987438624760aa570ab6359e164921f5fe","pr_number":7292,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110683406529466368","occurred_at":"2026-10-15T10:45:13Z","signature_id":"8a06a2b28a3f5de3ef2896d9d00f7a6eabc2ba94528c7b161f11b59200aac1d9","pr_number":7387,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110669729159974912","occurred_at":"2026-10-15T09:50:52Z","signature_id":"909a84cc2e92a28688b23dd0fb6c6b783bf49f7dc2fe286fe4279d7b42f6e844","pr_number":7502,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7487/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110607153617178624","occurred_at":"2026-10-15T05:42:13Z","signature_id":"97bfd8c685ed25785b98d0c464723104c6310fb126eedbfee4754118c08c6c7e","pr_number":7487,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558373865852928","occurred_at":"2026-10-15T02:28:23Z","signature_id":"6a6340d98166407ddced8d99afdbb8460d70e92d4881d4189e7a029a2ec7ed6f","pr_number":7524,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7538/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110474243606056960","occurred_at":"2026-10-14T20:54:04Z","signature_id":"2edb65bb26c1fcf13b3d6493ff98a75094606084998e2774dd1fdd5d9eea40d3","pr_number":7538,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110436010226618368","occurred_at":"2026-10-14T18:22:06Z","signature_id":"2bcbcddc1129669cc16a652effa6b618e89105ac18bfc5c7045a63681a784df7","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7525/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110338041116626944","occurred_at":"2026-10-14T11:52:51Z","signature_id":"ef7bc72ecbccb9636abbff125ef303658b2a72e4790bc2dc66060294d79a7d66","pr_number":7525,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110305876723109888","occurred_at":"2026-10-14T09:44:59Z","signature_id":"0f15a710eff8933fff6957af6014aa8dd4f7920e26e97a6c196fb7ab83a7a410","pr_number":7497,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110168566228586496","occurred_at":"2026-10-14T00:39:25Z","signature_id":"ddbe6c18bc9334463938901bbf610b5d5f5e544a53db6228b02093b131657e11","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110123884639424512","occurred_at":"2026-10-13T21:41:52Z","signature_id":"7abba32d1386c404a65777a77a6155bf3cb76d06da7c18bd2cc1819fac47609a","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110115702512816128","occurred_at":"2026-10-13T21:09:22Z","signature_id":"3a7b7f1faaa1631909c7c15abf2900e7c8f3a919926de8c9c9b43b605988f0ed","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110112563529256960","occurred_at":"2026-10-13T20:56:53Z","signature_id":"814ddd39348331a533c35437b54ed67f25a7378b56607756b2224dbb1c825092","pr_number":7469,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110035842520059904","occurred_at":"2026-10-13T15:52:02Z","signature_id":"04454a362557e528929a3344b4548e2c84ce0874d9f75e7bb99b09f4d24248b9","pr_number":7313,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110034877993717760","occurred_at":"2026-10-13T15:48:07Z","signature_id":"704940cf6bb6426bf6cc6c262b56027ce5f8ecb93098d3e28c4138854cfe377f","pr_number":7503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110013226245689344","occurred_at":"2026-10-13T14:22:09Z","signature_id":"45e5d5f08ac9de30d727c5c8b1bb049fe1a2cc352bd7a84c7804f7b87b200c41","pr_number":7480,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109804957828911104","occurred_at":"2026-10-13T00:34:34Z","signature_id":"241100486d1e2e5f6bf10dfd23c29984ebe1db934e3f4042fdf918032b8eac38","pr_number":7485,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7311/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109657273612636160","occurred_at":"2026-10-12T14:47:36Z","signature_id":"8a2cb86eeaaac2be5c2b729648eafebf2a78774b306792068dd64d9c83e1f663","pr_number":7311,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109536551896616960","occurred_at":"2026-10-12T06:48:01Z","signature_id":"de4fcc56ca6c7e3d2de039fd00e411d285c0c7d0026255cd09fc37bbc4e1d4c7","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109474078921330688","occurred_at":"2026-10-12T02:39:47Z","signature_id":"7e05f12cb3af3d2df3800593d5e4d70e1bcc0ddd1495a395607587d40f14bb85","pr_number":7439,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":16,"runs_affected":16,"failed_runs":16,"run_impact_percent":3.258655804480652,"anchor_occurrences":16,"after_last_push_count":7,"trend_counts":[0,2,3,2,7,2,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":16,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":6}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5mfh5m\" for cluster \"cluster-candidate-4-23-5mfh5m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x83c94b64840\u003e: \n    failed to create NodePool np-5mfh5m: failed waiting for nodepool=\"np-5mfh5m\" for cluster \"cluster-candidate-4-23-5mfh5m\" in resourcegroup=\"rg-candidate-4-23-t8lk4npslqvb\" to finish creating: GET https://rp.j3075072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4ba6beec-b4a8-4b19-9703-a8d4018563a9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4ba6beec-b4a8-4b19-9703-a8d4018563a9\",\n      \"name\": \"4ba6beec-b4a8-4b19-9703-a8d4018563a9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T10:16:52.661682642Z\",\n      \"endTime\": \"2026-10-16T10:35:56.734294404Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t8lk4npslqvb-candidate-5mfh5m--managed/providers/Microsoft.Compute/virtualMachines/w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t8lk4npslqvb-candidate-5mfh5m--managed/providers/Microsoft.Compute/virtualMachines/w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3075072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4ba6beec-b4a8-4b19-9703-a8d4018563a9\n            correlation ID: 2de83bb6-2187-443d-9a87-45e45c14da85\n            client request ID: 229ecc64-817b-4213-bc11-c707f778c1d6\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5mfh5m\" for cluster \"cluster-candidate-4-23-5mfh5m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x83c94b64840\u003e: \n    failed to create NodePool np-5mfh5m: failed waiting for nodepool=\"np-5mfh5m\" for cluster \"cluster-candidate-4-23-5mfh5m\" in resourcegroup=\"rg-candidate-4-23-t8lk4npslqvb\" to finish creating: GET https://rp.j3075072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4ba6beec-b4a8-4b19-9703-a8d4018563a9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4ba6beec-b4a8-4b19-9703-a8d4018563a9\",\n      \"name\": \"4ba6beec-b4a8-4b19-9703-a8d4018563a9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T10:16:52.661682642Z\",\n      \"endTime\": \"2026-10-16T10:35:56.734294404Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t8lk4npslqvb-candidate-5mfh5m--managed/providers/Microsoft.Compute/virtualMachines/w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t8lk4npslqvb-candidate-5mfh5m--managed/providers/Microsoft.Compute/virtualMachines/w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3075072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4ba6beec-b4a8-4b19-9703-a8d4018563a9\n            correlation ID: 2de83bb6-2187-443d-9a87-45e45c14da85\n            client request ID: 229ecc64-817b-4213-bc11-c707f778c1d6\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3408412f86c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-nvxdbn5qmgtb\" to finish creating: GET https://rp.j6657920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a18fbb87-8bb7-4a74-8f21-e96af9bfcf16\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a18fbb87-8bb7-4a74-8f21-e96af9bfcf16\",\n      \"name\": \"a18fbb87-8bb7-4a74-8f21-e96af9bfcf16\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T08:39:54.399138921Z\",\n      \"endTime\": \"2026-10-16T08:58:56.325845484Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nvxdbn5qmgtb--managed/providers/Microsoft.Compute/virtualMachines/p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nvxdbn5qmgtb--managed/providers/Microsoft.Compute/virtualMachines/p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6657920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a18fbb87-8bb7-4a74-8f21-e96af9bfcf16\n            correlation ID: 956ae344-8430-4fa1-b7b7-d3523f2ec507\n            client request ID: 149dc074-a954-4585-b9e7-50faa51cea7a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3408412f86c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-nvxdbn5qmgtb\" to finish creating: GET https://rp.j6657920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a18fbb87-8bb7-4a74-8f21-e96af9bfcf16\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a18fbb87-8bb7-4a74-8f21-e96af9bfcf16\",\n      \"name\": \"a18fbb87-8bb7-4a74-8f21-e96af9bfcf16\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T08:39:54.399138921Z\",\n      \"endTime\": \"2026-10-16T08:58:56.325845484Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nvxdbn5qmgtb--managed/providers/Microsoft.Compute/virtualMachines/p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nvxdbn5qmgtb--managed/providers/Microsoft.Compute/virtualMachines/p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6657920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a18fbb87-8bb7-4a74-8f21-e96af9bfcf16\n            correlation ID: 956ae344-8430-4fa1-b7b7-d3523f2ec507\n            client request ID: 149dc074-a954-4585-b9e7-50faa51cea7a\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x39a645494420\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-tfxpkjkjrqvw\" to finish creating: GET https://rp.j8228736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54644c94-7f95-4829-9f03-e9b939189ba9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54644c94-7f95-4829-9f03-e9b939189ba9\",\n      \"name\": \"54644c94-7f95-4829-9f03-e9b939189ba9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T15:18:30.534168337Z\",\n      \"endTime\": \"2026-10-15T15:47:34.762349229Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q2n5p1l1g7q3y7a-np-1-6mdqn-p62bb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-tfxpkjkjrqvw--managed/providers/Microsoft.Compute/virtualMachines/q2n5p1l1g7q3y7a-np-1-6mdqn-p62bb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q2n5p1l1g7q3y7a-np-1-6mdqn-p62bb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-tfxpkjkjrqvw--managed/providers/Microsoft.Compute/virtualMachines/q2n5p1l1g7q3y7a-np-1-6mdqn-p62bb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8228736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54644c94-7f95-4829-9f03-e9b939189ba9\n            correlation ID: af51592b-1bd9-4ddd-81a5-d7c702c77eb4\n            client request ID: e54f4800-8170-46d4-a0fb-e016b4fc8317\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x39a645494420\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-tfxpkjkjrqvw\" to finish creating: GET https://rp.j8228736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54644c94-7f95-4829-9f03-e9b939189ba9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54644c94-7f95-4829-9f03-e9b939189ba9\",\n      \"name\": \"54644c94-7f95-4829-9f03-e9b939189ba9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T15:18:30.534168337Z\",\n      \"endTime\": \"2026-10-15T15:47:34.762349229Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q2n5p1l1g7q3y7a-np-1-6mdqn-p62bb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-tfxpkjkjrqvw--managed/providers/Microsoft.Compute/virtualMachines/q2n5p1l1g7q3y7a-np-1-6mdqn-p62bb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q2n5p1l1g7q3y7a-np-1-6mdqn-p62bb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-tfxpkjkjrqvw--managed/providers/Microsoft.Compute/virtualMachines/q2n5p1l1g7q3y7a-np-1-6mdqn-p62bb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8228736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54644c94-7f95-4829-9f03-e9b939189ba9\n            correlation ID: af51592b-1bd9-4ddd-81a5-d7c702c77eb4\n            client request ID: e54f4800-8170-46d4-a0fb-e016b4fc8317\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7483/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111022552083075072","occurred_at":"2026-10-16T09:12:51Z","signature_id":"ed2036aae2cf00dc5018c7a58da50f85fd92d64f0bb21b758adcb9a18d2da21c","pr_number":7483,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110999442776657920","occurred_at":"2026-10-16T07:41:02Z","signature_id":"4e0ccb0d44ba37e343acca65141914d09bdba66d4aea11c0a4acc789d41b1f7b","pr_number":7566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7530/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110735645818228736","occurred_at":"2026-10-15T14:12:39Z","signature_id":"6fecd2e379b5b970f9431a5a8c59ead1664da099e1d0579291569197994638d2","pr_number":7530,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110735571163811840","occurred_at":"2026-10-15T14:12:30Z","signature_id":"c8d0ee720bdf2081b6a5273c3aaece4bd87951ddeb0f90450369fd27dc7d7c31","pr_number":7560,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7489/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110733173536067584","occurred_at":"2026-10-15T14:02:58Z","signature_id":"26d3525c854c8f4d471bc29f8a07cb7f55e7964f9309e9ea29f0216490bc6a62","pr_number":7489,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110728083584585728","occurred_at":"2026-10-15T13:42:45Z","signature_id":"116b4334c4a9fafad2546205e2a5dac1b635d9b46b6b9d1482ca71ca3eb32159","pr_number":7387,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110717219770994688","occurred_at":"2026-10-15T12:59:33Z","signature_id":"747f466235d4bddbe01ad4b4f61b790928abd8334698fea467aa72ac3adc35ed","pr_number":7373,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110634495622254592","occurred_at":"2026-10-15T07:30:41Z","signature_id":"31f191c92dde73db30b06f9f049ccf97491546066d75fece41a85257fdc973e3","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7532/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110619354365169664","occurred_at":"2026-10-15T06:30:42Z","signature_id":"94868267fe486231af0656c22709b87554974909acf8bc79f7caf9dd836318e5","pr_number":7532,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7515/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110290387544313856","occurred_at":"2026-10-14T08:43:30Z","signature_id":"50e6f316011a8296fa2af306e18ddc7dbd6a5f66fbf21de99e30ffc49bba1879","pr_number":7515,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110267140211740672","occurred_at":"2026-10-14T07:11:07Z","signature_id":"6b14ed8a4511df290f8bf9163c6eafbbcf710dd60c50bd84885e6283a1516ec7","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110034877993717760","occurred_at":"2026-10-13T15:48:07Z","signature_id":"0b110e5c27c96f6c961d41c663c021306587e891bbe40929150d9ebda913bc3f","pr_number":7503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109994317174018048","occurred_at":"2026-10-13T13:07:01Z","signature_id":"64c28e7e5c45c72d72e63cae7d8453adeb4e0d6a5ad577f87fac746a2ba5e16b","pr_number":7488,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109933810454892544","occurred_at":"2026-10-13T09:06:35Z","signature_id":"719b91552139342f4f6f074922a7af2b8c76f3cf3df0a249b92551325953e60c","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109754183480512512","occurred_at":"2026-10-12T21:12:49Z","signature_id":"e7d1ae872e2d1e1f299e0950e3c7121739a7ca3253cb34361fe8fe1de4b27d3a","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109709852522582016","occurred_at":"2026-10-12T18:16:39Z","signature_id":"f07f78c84cfe8e58c2511543b04e0ca7a20c652e502f4d4a311097ea89b1549b","pr_number":7177,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":15,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.814663951120163,"anchor_occurrences":15,"after_last_push_count":101,"trend_counts":[0,3,12,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":162,"prior_last_seen_at":"2026-10-09T11:00:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x31b87eaf00d8\u003e: \n    resource group \"node-limit-test-2ch85v6j6t94\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2ch85v6j6t94\" to finish deleting: GET https://rp.j9475584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2de5bbd0-8d23-4d85-8e58-9564e3649e84\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2de5bbd0-8d23-4d85-8e58-9564e3649e84\",\n      \"name\": \"2de5bbd0-8d23-4d85-8e58-9564e3649e84\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T16:37:18.781083053Z\",\n      \"endTime\": \"2026-10-13T17:01:24.31591322Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0s3j58fj1f3fm4vsgrm1lrp3it3l1 still exists (deletion dispatched at 2026-10-13T16:37:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9475584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2de5bbd0-8d23-4d85-8e58-9564e3649e84\n            correlation ID: 879b6302-da33-4ea2-941e-e29845fe6259\n            client request ID: f5a45ff4-173b-4f67-a8d6-7bf91987cdf6\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x31b87eaf00d8\u003e: \n    resource group \"node-limit-test-2ch85v6j6t94\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2ch85v6j6t94\" to finish deleting: GET https://rp.j9475584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2de5bbd0-8d23-4d85-8e58-9564e3649e84\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2de5bbd0-8d23-4d85-8e58-9564e3649e84\",\n      \"name\": \"2de5bbd0-8d23-4d85-8e58-9564e3649e84\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T16:37:18.781083053Z\",\n      \"endTime\": \"2026-10-13T17:01:24.31591322Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0s3j58fj1f3fm4vsgrm1lrp3it3l1 still exists (deletion dispatched at 2026-10-13T16:37:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9475584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2de5bbd0-8d23-4d85-8e58-9564e3649e84\n            correlation ID: 879b6302-da33-4ea2-941e-e29845fe6259\n            client request ID: f5a45ff4-173b-4f67-a8d6-7bf91987cdf6\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x3bf37649c048\u003e: \n    resource group \"customer-rg-kptnhtffptgl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-kptnhtffptgl\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\",\n      \"name\": \"d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T16:42:39.455530141Z\",\n      \"endTime\": \"2026-10-13T17:06:41.723960509Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0oaptqtlfm45pi2ru69verofh93nj still exists (deletion dispatched at 2026-10-13T16:42:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\n            correlation ID: 57954ad9-2ba3-4943-8c75-f5745f7ae1f7\n            client request ID: bbb4049f-9eb5-4ee1-9bdd-ccda4efc41c1\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x3bf37649c048\u003e: \n    resource group \"customer-rg-kptnhtffptgl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-kptnhtffptgl\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\",\n      \"name\": \"d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T16:42:39.455530141Z\",\n      \"endTime\": \"2026-10-13T17:06:41.723960509Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0oaptqtlfm45pi2ru69verofh93nj still exists (deletion dispatched at 2026-10-13T16:42:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\n            correlation ID: 57954ad9-2ba3-4943-8c75-f5745f7ae1f7\n            client request ID: bbb4049f-9eb5-4ee1-9bdd-ccda4efc41c1\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x309531b0198\u003e: \n    resource group \"ea-list-wtljvtmmhcsk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-list8h4wvd\" in resourcegroup=\"ea-list-wtljvtmmhcsk\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2f74cf89-c5af-4bbc-9410-e14470aca714\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2f74cf89-c5af-4bbc-9410-e14470aca714\",\n      \"name\": \"2f74cf89-c5af-4bbc-9410-e14470aca714\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T16:38:18.149454561Z\",\n      \"endTime\": \"2026-10-13T17:02:21.62230875Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o6snbp7v915nacod7cib95rmn96j still exists (deletion dispatched at 2026-10-13T16:38:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2f74cf89-c5af-4bbc-9410-e14470aca714\n            correlation ID: 8d339200-f0e0-47b8-90d7-53a61a990c91\n            client request ID: 742fa150-0903-4156-b9ea-5d97b505f86c\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x309531b0198\u003e: \n    resource group \"ea-list-wtljvtmmhcsk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-list8h4wvd\" in resourcegroup=\"ea-list-wtljvtmmhcsk\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2f74cf89-c5af-4bbc-9410-e14470aca714\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2f74cf89-c5af-4bbc-9410-e14470aca714\",\n      \"name\": \"2f74cf89-c5af-4bbc-9410-e14470aca714\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T16:38:18.149454561Z\",\n      \"endTime\": \"2026-10-13T17:02:21.62230875Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o6snbp7v915nacod7cib95rmn96j still exists (deletion dispatched at 2026-10-13T16:38:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2f74cf89-c5af-4bbc-9410-e14470aca714\n            correlation ID: 8d339200-f0e0-47b8-90d7-53a61a990c91\n            client request ID: 742fa150-0903-4156-b9ea-5d97b505f86c\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7489/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110035013809475584","occurred_at":"2026-10-13T15:48:44Z","signature_id":"ed688746e174fb91b628587af9f6d4fc2c44f7a9d83bdae64d4e89a50cd9e76a","pr_number":7489,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"022aadd1c5b43f5fcb46b49de5bea32873432564b44b704b42ed859546fe8e82","pr_number":7496,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"6d01e4638c3973f17302f07ed79508b57643e22d74bb892cac9c1af52e92a292","pr_number":7496,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"72c1ad55e4062aec518f3f62817b93e6395a0432a25afcb61cb220d43060d265","pr_number":7496,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"7be575f2f85e63d8ce2308f15291aa10c120d9bccad74cdd55d95042f97a55a5","pr_number":7496,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"94aae066f49ff828f8658c561f57e2a599bd9f4bcee2e2d506ad8011ab4f1278","pr_number":7496,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"a17143f3d84444b29b8010bee708abc0adf653cb05b16055bea351d2e9fb5d99","pr_number":7496,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"b92ba843e0e832a2ad56a0bbd96074f26f53e9816edef7f8e3b6ac20d8300aff","pr_number":7496,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"bf1c9ad85dc67c52bb5db81e418c326fc84cc55cd5304ca050cd7f99526fd13a","pr_number":7496,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"d91ceb0c461dd8cc9e4f63cf5a15d575daca06c95808c7ef0af47faf649a13d6","pr_number":7496,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"e5551334aa432c7f1d4fcaa4df0fbb0de936bcd598badb4046f07fb57beb7449","pr_number":7496,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"ecc5ff218b668fca411a6a387f8fa1e8cc8fcb2c2b7b038bd92d4f9b96814b90","pr_number":7496,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109659008674566144","occurred_at":"2026-10-12T14:54:37Z","signature_id":"aa1b645215452d614b0ab2fb286d0f4c2a52df3c3bf4b0684396494f99dc7271","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","signature_id":"9464cf8289c40c4ace5feeaa2af2e60f2ff16f8a6b1306bce054078519a3b607","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","signature_id":"c8fe87bb1fbea1d2b82daec5c2abcf5f379166a6c5ecbc60a09403cacb34657b","pr_number":6920,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":13,"runs_affected":10,"failed_runs":10,"run_impact_percent":2.0366598778004072,"anchor_occurrences":13,"after_last_push_count":13,"trend_counts":[0,3,6,3,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":36,"prior_last_seen_at":"2026-10-09T16:02:30Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-6xdv46kj2t6c/cluster-candidate-4-23-ds6g6p should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x100c616f6120\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-ds6g6p: failed waiting for cluster=\"cluster-candidate-4-23-ds6g6p\" in resourcegroup=\"rg-candidate-4-23-6xdv46kj2t6c\" to finish creating: GET https://rp.j0840320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f51531d5-d416-4611-a4d7-f0f2d7392927\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f51531d5-d416-4611-a4d7-f0f2d7392927\",\n      \"name\": \"f51531d5-d416-4611-a4d7-f0f2d7392927\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T17:08:33.97123397Z\",\n      \"endTime\": \"2026-10-15T17:37:33.800671635Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0840320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f51531d5-d416-4611-a4d7-f0f2d7392927\n            correlation ID: b723ee7e-f41a-48af-8379-6cf9b8adcb05\n            client request ID: a255119f-4ee0-4d8e-8cd6-e4350ddd6c3d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-6xdv46kj2t6c/cluster-candidate-4-23-ds6g6p should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x100c616f6120\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-ds6g6p: failed waiting for cluster=\"cluster-candidate-4-23-ds6g6p\" in resourcegroup=\"rg-candidate-4-23-6xdv46kj2t6c\" to finish creating: GET https://rp.j0840320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f51531d5-d416-4611-a4d7-f0f2d7392927\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f51531d5-d416-4611-a4d7-f0f2d7392927\",\n      \"name\": \"f51531d5-d416-4611-a4d7-f0f2d7392927\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T17:08:33.97123397Z\",\n      \"endTime\": \"2026-10-15T17:37:33.800671635Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0840320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f51531d5-d416-4611-a4d7-f0f2d7392927\n            correlation ID: b723ee7e-f41a-48af-8379-6cf9b8adcb05\n            client request ID: a255119f-4ee0-4d8e-8cd6-e4350ddd6c3d\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0x24524eb9ea50\u003e: \n    GET https://rp.j0214528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/812fe66b-c9fc-46f9-8d46-bf8a0cd5068b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/812fe66b-c9fc-46f9-8d46-bf8a0cd5068b\",\n      \"name\": \"812fe66b-c9fc-46f9-8d46-bf8a0cd5068b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:04:33.551295422Z\",\n      \"endTime\": \"2026-10-14T23:33:41.530060174Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0214528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/812fe66b-c9fc-46f9-8d46-bf8a0cd5068b\n            correlation ID: d14d5694-03de-4ac8-b587-4d0e35762d84\n            client request ID: 5cfd8941-3a60-4ebe-8bc6-a659901f1339\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0x24524eb9ea50\u003e: \n    GET https://rp.j0214528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/812fe66b-c9fc-46f9-8d46-bf8a0cd5068b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/812fe66b-c9fc-46f9-8d46-bf8a0cd5068b\",\n      \"name\": \"812fe66b-c9fc-46f9-8d46-bf8a0cd5068b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T23:04:33.551295422Z\",\n      \"endTime\": \"2026-10-14T23:33:41.530060174Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0214528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/812fe66b-c9fc-46f9-8d46-bf8a0cd5068b\n            correlation ID: d14d5694-03de-4ac8-b587-4d0e35762d84\n            client request ID: 5cfd8941-3a60-4ebe-8bc6-a659901f1339\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0x225a34843540\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5rp2rtlm9fj9\" to finish creating: GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/680a3372-b50b-4c21-a07b-0256c4145619\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/680a3372-b50b-4c21-a07b-0256c4145619\",\n      \"name\": \"680a3372-b50b-4c21-a07b-0256c4145619\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T09:49:54.805487688Z\",\n      \"endTime\": \"2026-10-14T10:18:57.194996434Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/680a3372-b50b-4c21-a07b-0256c4145619\n            correlation ID: 5895723a-3ff0-431a-a927-5b6e194fc741\n            client request ID: 350b7a37-c335-451c-b2e9-99a3b665afa1\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0x225a34843540\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5rp2rtlm9fj9\" to finish creating: GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/680a3372-b50b-4c21-a07b-0256c4145619\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/680a3372-b50b-4c21-a07b-0256c4145619\",\n      \"name\": \"680a3372-b50b-4c21-a07b-0256c4145619\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T09:49:54.805487688Z\",\n      \"endTime\": \"2026-10-14T10:18:57.194996434Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/680a3372-b50b-4c21-a07b-0256c4145619\n            correlation ID: 5895723a-3ff0-431a-a927-5b6e194fc741\n            client request ID: 350b7a37-c335-451c-b2e9-99a3b665afa1\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7301/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110767880390840320","occurred_at":"2026-10-15T16:20:40Z","signature_id":"d48e64ebf888d9dd74307cf3b5bc70c5e2ff427649dd00d12b73ed878850e2ef","pr_number":7301,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110494690280214528","occurred_at":"2026-10-14T22:15:16Z","signature_id":"ed9fcd46de5fd368c22dffbc0a6841cff99957b615f7c90f7722dcf43f30b6fb","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110291728471691264","occurred_at":"2026-10-14T08:48:48Z","signature_id":"801661c2c5d4111b3b2777864164064d45ed90a83f09dd5139dcd40652bab0e6","pr_number":7500,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110213919732666368","occurred_at":"2026-10-14T03:39:38Z","signature_id":"bccfe50993c9da8f2e5a003a505f4895a9e3eda1fc47df0e84d9a7dfd246b05a","pr_number":7509,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110131584773394432","occurred_at":"2026-10-13T22:12:28Z","signature_id":"3b8fba5a1aa06016e6c6895bc3863f7f6cf359856948127e7c81a165583cfcd6","pr_number":7509,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"077f299c2a7977d35eb0dc0f0f3806d25b08b958ea3c8d1b9f28717ecf19fa10","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"48f96c794b4248e434313e5383dab85570d91b004b93d8173274ca3b89e3bd5b","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"61bc9fb2baad09f02ca746a494c19b296e155e5562e460538f3b5c55bfa4f3f6","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"afbbb0fd16401b97b4a184026640540584d09079290abbba4ab2babc53541b3a","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110027575299411968","occurred_at":"2026-10-13T15:15:38Z","signature_id":"5c9c535151da7212f6c377a91341a3089f208d25d42e32c59743276d5adb4a39","pr_number":7485,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109715328622661632","occurred_at":"2026-10-12T18:38:25Z","signature_id":"ba9594d183dcf679de486f3ef983cb99f7de22b5534227c654d1211082cc3ca0","pr_number":7479,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"614ca0e1ea00ef1b4414e6ab8b3a11a8b2da5132a3da64d8b9faf88c1203affd","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","signature_id":"026e1543f757852d519080f003f4d7ebec18027795d5835ee8f594400d820c5e","pr_number":7463,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":12,"runs_affected":11,"failed_runs":11,"run_impact_percent":2.240325865580448,"anchor_occurrences":12,"after_last_push_count":12,"trend_counts":[1,1,1,4,5,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":20,"prior_last_seen_at":"2026-10-11T23:29:37Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x340dc62fa018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-dsg52jhcqb45\" to finish creating: GET https://rp.j7338752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baecf025-dfba-4e06-b1e1-28ab3b4cbfe6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baecf025-dfba-4e06-b1e1-28ab3b4cbfe6\",\n      \"name\": \"baecf025-dfba-4e06-b1e1-28ab3b4cbfe6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T14:26:03.913560661Z\",\n      \"endTime\": \"2026-10-16T14:45:08.626305681Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7338752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baecf025-dfba-4e06-b1e1-28ab3b4cbfe6\n            correlation ID: 85e240f2-d0e4-4c29-bd30-d4d4327881c2\n            client request ID: d5c7b389-b2c6-4aba-9dfa-9055479c941f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x340dc62fa018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-dsg52jhcqb45\" to finish creating: GET https://rp.j7338752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baecf025-dfba-4e06-b1e1-28ab3b4cbfe6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baecf025-dfba-4e06-b1e1-28ab3b4cbfe6\",\n      \"name\": \"baecf025-dfba-4e06-b1e1-28ab3b4cbfe6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T14:26:03.913560661Z\",\n      \"endTime\": \"2026-10-16T14:45:08.626305681Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7338752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baecf025-dfba-4e06-b1e1-28ab3b4cbfe6\n            correlation ID: 85e240f2-d0e4-4c29-bd30-d4d4327881c2\n            client request ID: d5c7b389-b2c6-4aba-9dfa-9055479c941f\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x12c08c2c7ca0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-ftltmptlj2wv\" to finish creating: GET https://rp.j5160960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a570a939-3665-4ef5-997f-a303cdd226fe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a570a939-3665-4ef5-997f-a303cdd226fe\",\n      \"name\": \"a570a939-3665-4ef5-997f-a303cdd226fe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T21:24:37.707389453Z\",\n      \"endTime\": \"2026-10-15T21:53:37.621733074Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5160960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a570a939-3665-4ef5-997f-a303cdd226fe\n            correlation ID: f3ada34b-c475-444b-90fd-a47f28fcd020\n            client request ID: 8db3b288-30a6-4fd1-8aaf-660ebcafd494\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x12c08c2c7ca0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-ftltmptlj2wv\" to finish creating: GET https://rp.j5160960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a570a939-3665-4ef5-997f-a303cdd226fe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a570a939-3665-4ef5-997f-a303cdd226fe\",\n      \"name\": \"a570a939-3665-4ef5-997f-a303cdd226fe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T21:24:37.707389453Z\",\n      \"endTime\": \"2026-10-15T21:53:37.621733074Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5160960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a570a939-3665-4ef5-997f-a303cdd226fe\n            correlation ID: f3ada34b-c475-444b-90fd-a47f28fcd020\n            client request ID: 8db3b288-30a6-4fd1-8aaf-660ebcafd494\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x16cbbb3841e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-qc5d6jswb5qj\" to finish creating: GET https://rp.j9447936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc6a40ed-7014-400d-bad8-400eddcb5f30\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc6a40ed-7014-400d-bad8-400eddcb5f30\",\n      \"name\": \"dc6a40ed-7014-400d-bad8-400eddcb5f30\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T18:11:06.765137924Z\",\n      \"endTime\": \"2026-10-15T18:40:10.370410387Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9447936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc6a40ed-7014-400d-bad8-400eddcb5f30\n            correlation ID: 36d679d4-7225-44c2-8efc-12d1c6c12f83\n            client request ID: 0fff6042-c3bb-454b-84b5-80c6919f7c90\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x16cbbb3841e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-qc5d6jswb5qj\" to finish creating: GET https://rp.j9447936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc6a40ed-7014-400d-bad8-400eddcb5f30\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc6a40ed-7014-400d-bad8-400eddcb5f30\",\n      \"name\": \"dc6a40ed-7014-400d-bad8-400eddcb5f30\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T18:11:06.765137924Z\",\n      \"endTime\": \"2026-10-15T18:40:10.370410387Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9447936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc6a40ed-7014-400d-bad8-400eddcb5f30\n            correlation ID: 36d679d4-7225-44c2-8efc-12d1c6c12f83\n            client request ID: 0fff6042-c3bb-454b-84b5-80c6919f7c90\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111084766907338752","occurred_at":"2026-10-16T13:20:05Z","signature_id":"af2923d29e8301ef3cd00e5f05932057532b4d49d8d8bcf7e33cfa90a1f4a5a8","pr_number":7582,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110825638725160960","occurred_at":"2026-10-15T20:10:18Z","signature_id":"f30e9942973b7267339039965604c29f853812f4919f100cf7e81ea17ccde3fe","pr_number":6746,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110779900129447936","occurred_at":"2026-10-15T17:08:38Z","signature_id":"bbcbcfaf4836e745643388f06f9a99888d842245d9e059d50795eafd0a991ec3","pr_number":7387,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110779900129447936","occurred_at":"2026-10-15T17:08:38Z","signature_id":"f69b20c9861eedd0b141eeba6c3bfdb4421cb084e340f59c6e991486db929c6a","pr_number":7387,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110772387883520000","occurred_at":"2026-10-15T16:38:43Z","signature_id":"1794f836ef670c568ca7792172e3877cb4f26e64a13ea2c0be7db90a261119c6","pr_number":7560,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110661117289172992","occurred_at":"2026-10-15T09:16:33Z","signature_id":"06863ec0e9de510178dcfdfbf59d8b072eb6aaa6dc4141168ebda267b24b0e5d","pr_number":7524,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6866/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110463708953055232","occurred_at":"2026-10-14T20:12:13Z","signature_id":"cecf631ae361f8a51d4b72842068b68b69f80342291c1cbf1990a183f6131c1e","pr_number":6866,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110377553440018432","occurred_at":"2026-10-14T14:29:52Z","signature_id":"37191323f61b58a1822c2e181e6491d477ddd25ec6e827fd594606032d08cfcf","pr_number":7529,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110361307805913088","occurred_at":"2026-10-14T13:25:18Z","signature_id":"0aa02a14101484db576124bfb9226ae693d8ca3ab3cc3bef546f2e12d693deba","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110334779541950464","occurred_at":"2026-10-14T11:39:53Z","signature_id":"2fcc5bf330faefb96ced672e9367aef3eab81cd45e1be28037ce9e2ea61effd3","pr_number":7519,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109797931459547136","occurred_at":"2026-10-13T00:06:39Z","signature_id":"43851cc86df3904c329cff3884669981de4ca71371d688f448f3b74e28255f14","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109708565542342656","occurred_at":"2026-10-12T18:11:33Z","signature_id":"8d1213d4562f39353e445c7a58b0fd1c57fd1080920c955cf4d994b8f4a3de3c","pr_number":7373,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ecdbfeab4fd2cbaa5fe8d221e09bffe86dfcfd6d05ac80416e0ba46ab9b719fc","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"shared-read-only\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should get an existing HCP cluster and report a missing cluster through the RP","occurrences":12,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.6109979633401222,"anchor_occurrences":12,"after_last_push_count":0,"trend_counts":[0,12,0,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should get an existing HCP cluster and report a missing cluster through the RP","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"External auth issuer validation rejects an invalid external auth issuer URL without leaving a child or changing the shared cluster","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Immutable cluster configuration rejects an immutable cluster DNS prefix PATCH without changing the shared cluster","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Immutable node pool configuration rejects an immutable node pool OS disk size PATCH without changing the shared node pool","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Node pool create validation rejects a negative replica count without creating a node pool or changing the shared cluster","occurrences":1}],"full_error_samples":["provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-v4t896dszvwt\": PUT https://rp.j3300864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-v4t896dszvwt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"MultipleErrorsOccurred\",\n    \"message\": \"2 errors occurred\",\n    \"details\": [\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Too long: may not be more than 15 bytes\",\n        \"target\": \"id\"\n      },\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n        \"target\": \"id\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-v4t896dszvwt\": PUT https://rp.j3300864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-v4t896dszvwt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"MultipleErrorsOccurred\",\n    \"message\": \"2 errors occurred\",\n    \"details\": [\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Too long: may not be more than 15 bytes\",\n        \"target\": \"id\"\n      },\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n        \"target\": \"id\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------","provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-vlx4fgdsmn4s\": PUT https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-vlx4fgdsmn4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"MultipleErrorsOccurred\",\n    \"message\": \"2 errors occurred\",\n    \"details\": [\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Too long: may not be more than 15 bytes\",\n        \"target\": \"id\"\n      },\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n        \"target\": \"id\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-vlx4fgdsmn4s\": PUT https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-vlx4fgdsmn4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"MultipleErrorsOccurred\",\n    \"message\": \"2 errors occurred\",\n    \"details\": [\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Too long: may not be more than 15 bytes\",\n        \"target\": \"id\"\n      },\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n        \"target\": \"id\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------","provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-lr2d54k7v8w5\": PUT https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-lr2d54k7v8w5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"MultipleErrorsOccurred\",\n    \"message\": \"2 errors occurred\",\n    \"details\": [\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Too long: may not be more than 15 bytes\",\n        \"target\": \"id\"\n      },\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n        \"target\": \"id\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-lr2d54k7v8w5\": PUT https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-lr2d54k7v8w5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"MultipleErrorsOccurred\",\n    \"message\": \"2 errors occurred\",\n    \"details\": [\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Too long: may not be more than 15 bytes\",\n        \"target\": \"id\"\n      },\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n        \"target\": \"id\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109616618123300864","occurred_at":"2026-10-12T12:06:11Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7471,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109616618123300864","occurred_at":"2026-10-12T12:06:11Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7471,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109616618123300864","occurred_at":"2026-10-12T12:06:11Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7471,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109616618123300864","occurred_at":"2026-10-12T12:06:11Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7471,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7470,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AzureKeyVaultKmsValidateIdentityPermissionCustomerError; detail message The identity does not have keys encrypt/decrypt permission on key vault \"\u003curl\u003e\"; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":10,"runs_affected":10,"failed_runs":10,"run_impact_percent":2.0366598778004072,"anchor_occurrences":10,"after_last_push_count":8,"trend_counts":[0,0,0,0,9,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":10,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":10}],"full_error_samples":["time=2026-10-16T10:37:16.131Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-16T10:37:17.576Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-16T10:37:18.879Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=182dddaef1eb2eeeaeb1c74f26cc60d61fef94e53fb090c30b83b72986af948d portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6554112-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F182dddaef1eb2eeeaeb1c74f26cc60d61fef94e53fb090c30b83b72986af948d\ntime=2026-10-16T10:39:19.372Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6554112-svc/providers/Microsoft.Resources/deployments/182dddaef1eb2eeeaeb1c74f26cc60d61fef94e53fb090c30b83b72986af948d/operationStatuses/08584094598472503244\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6554112-svc/providers/Microsoft.Resources/deployments/cluster-ismqifijm7koe\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6554112-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j6554112-svc\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\\\\\\\\\"https://ah-ci01-se-j6554112-1.vault.azure.net\\\\\\\\\\\\\\\"\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-10-15T22:37:03.691Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-15T22:37:05.354Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-15T22:37:06.762Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=bee630fb16cd45492e943a7ab4f37f451fa3266ef665a6d74f650467bd60800f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9396992-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fbee630fb16cd45492e943a7ab4f37f451fa3266ef665a6d74f650467bd60800f\ntime=2026-10-15T22:40:07.453Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9396992-svc/providers/Microsoft.Resources/deployments/bee630fb16cd45492e943a7ab4f37f451fa3266ef665a6d74f650467bd60800f/operationStatuses/08584095030595181402\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9396992-svc/providers/Microsoft.Resources/deployments/cluster-kpgg4tugptwaw\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9396992-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j9396992-svc\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\\\\\\\\\"https://ah-ci01-se-j9396992-1.vault.azure.net\\\\\\\\\\\\\\\"\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-10-15T20:48:50.569Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-15T20:48:52.496Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-15T20:48:54.249Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=8276bf3e2ac415cdaf52404f9c09132aafef64d9fdc663f4e7aca3248fabbce7 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3682688-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F8276bf3e2ac415cdaf52404f9c09132aafef64d9fdc663f4e7aca3248fabbce7\ntime=2026-10-15T20:51:24.632Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3682688-svc/providers/Microsoft.Resources/deployments/8276bf3e2ac415cdaf52404f9c09132aafef64d9fdc663f4e7aca3248fabbce7/operationStatuses/08584095095521617911\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3682688-svc/providers/Microsoft.Resources/deployments/cluster-a22genk7v3fe6\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3682688-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j3682688-svc\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\\\\\\\\\"https://ah-ci01-se-j3682688-1.vault.azure.net\\\\\\\\\\\\\\\"\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7576/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111038960766554112","occurred_at":"2026-10-16T10:18:04Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":7576,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110856138269396992","occurred_at":"2026-10-15T22:11:35Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":7387,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110825643053682688","occurred_at":"2026-10-15T20:10:19Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":7387,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110795115164340224","occurred_at":"2026-10-15T18:09:06Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6746,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7530/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110779961160765440","occurred_at":"2026-10-15T17:08:53Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":7530,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110767291346980864","occurred_at":"2026-10-15T16:18:33Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":7509,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7401/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110756585931280384","occurred_at":"2026-10-15T15:36:00Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":7401,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7527/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110738456727523328","occurred_at":"2026-10-15T14:23:58Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":7527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7489/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110712714677456896","occurred_at":"2026-10-15T12:41:40Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":7489,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110688954717048832","occurred_at":"2026-10-15T11:07:16Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":7554,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":10,"runs_affected":10,"failed_runs":10,"run_impact_percent":2.0366598778004072,"anchor_occurrences":10,"after_last_push_count":14,"trend_counts":[0,4,2,2,2,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":49,"prior_last_seen_at":"2026-10-09T22:55:44Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":10}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-15T17:33:59Z\n  Ended:       2026-10-15T18:00:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0840320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.184:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0840320/providers/microsoft.monitor/accounts/hcps-j0840320\", namespace=\"ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"69abfa76-e157-4c11-8f36-2d5d03c40035\"\n  Description: Pod ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-15T17:33:59Z\n  Ended:       2026-10-15T17:59:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0840320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.184:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0840320/providers/microsoft.monitor/accounts/hcps-j0840320\", namespace=\"ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"69abfa76-e157-4c11-8f36-2d5d03c40035\"\n  Description: Pod ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T04:10:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7932288-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.98:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7932288/providers/microsoft.monitor/accounts/hcps-j7932288\", namespace=\"ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w\", pod=\"cloud-network-config-controller-5dbf455969-n498c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"724a7794-4308-471d-8b59-9caf89cd21b6\"\n  Description: Pod ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w/cloud-network-config-controller-5dbf455969-n498c (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T04:10:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7932288-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.98:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7932288/providers/microsoft.monitor/accounts/hcps-j7932288\", namespace=\"ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w\", pod=\"cloud-network-config-controller-5dbf455969-n498c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"724a7794-4308-471d-8b59-9caf89cd21b6\"\n  Description: Pod ocm-arohcpci01-2tevnsff1cm4oc58rjaeu9o2afr8b41s-o1p7c9y0n1v9w4w/cloud-network-config-controller-5dbf455969-n498c (controller) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-14T23:26:27Z\n  Ended:       2026-10-14T23:56:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0214528-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0214528/providers/microsoft.monitor/accounts/hcps-j0214528\", namespace=\"ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e7a6870f-14b4-45e0-9c19-fce7f03813dc\"\n  Description: Pod ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-14T23:26:27Z\n  Ended:       2026-10-14T23:56:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0214528-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0214528/providers/microsoft.monitor/accounts/hcps-j0214528\", namespace=\"ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e7a6870f-14b4-45e0-9c19-fce7f03813dc\"\n  Description: Pod ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7301/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110767880390840320","occurred_at":"2026-10-15T16:20:40Z","signature_id":"fb61dba7bed400501acfa31e34ffb1e0063ef687ee57edad3920d058899b9f95","pr_number":7301,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558444497932288","occurred_at":"2026-10-15T02:28:40Z","signature_id":"ba8d8f3740082be57570aa73aa1e24accf3cf4552b7d0a87a0841840a1f288ed","pr_number":7492,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110494690280214528","occurred_at":"2026-10-14T22:15:16Z","signature_id":"4d0f56a4ffddc70a5d6865e72af328de9e6e01d601ffdd7f4779f43cf5ed097e","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110291728471691264","occurred_at":"2026-10-14T08:48:48Z","signature_id":"b11574c6b2c9e56fb0df8243603cc190494eabc1514bd622eab98ffa66345c88","pr_number":7500,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110131584773394432","occurred_at":"2026-10-13T22:12:28Z","signature_id":"ab3005e87ad04ff712d959705038bb2a3a5aadeb7a3d7406d2362bf8f67d8cf4","pr_number":7509,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110027575299411968","occurred_at":"2026-10-13T15:15:38Z","signature_id":"be3f2336b804bb116f4220473f3b514fdb2978302e5dac315a122f4afc722704","pr_number":7485,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109743222543618048","occurred_at":"2026-10-12T20:29:16Z","signature_id":"d4b2e29eb71d7ac5c954c156d7e1d4a7c4f83854fd89b2f4bf826b82158948d1","pr_number":7457,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109715328622661632","occurred_at":"2026-10-12T18:38:25Z","signature_id":"a25524a009142bcd67f70022a288b9e9be2947605657927be7457aeb02bcc1d2","pr_number":7479,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"ccfda79db149eab1089427f6b12af29fcb8d7b25163196bd92ebcab836f644a6","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","signature_id":"e9c8652d1165ca174ad5201f1063a5d60e6d9e1e776a81a9de4becc5e6a83ae5","pr_number":7463,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":9,"runs_affected":9,"failed_runs":9,"run_impact_percent":1.8329938900203666,"anchor_occurrences":9,"after_last_push_count":14,"trend_counts":[0,3,3,2,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":58,"prior_last_seen_at":"2026-10-08T04:44:42Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":9}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-15T17:31:05Z\n  Ended:       2026-10-15T17:56:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0840320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.184:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0840320/providers/microsoft.monitor/accounts/hcps-j0840320\", namespace=\"ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5z7k7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-15T17:31:05Z\n  Ended:       2026-10-15T18:01:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0840320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.184:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0840320/providers/microsoft.monitor/accounts/hcps-j0840320\", namespace=\"ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5z7k7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2tfbl4gqpolf3q5urat1gqkjg77m4lg4-a9g3b1n2x3r1g8j/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-14T23:34:33Z\n  Ended:       2026-10-14T23:51:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0214528-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0214528/providers/microsoft.monitor/accounts/hcps-j0214528\", namespace=\"ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-r2hbm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-14T23:35:30Z\n  Ended:       2026-10-14T23:52:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0214528-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0214528/providers/microsoft.monitor/accounts/hcps-j0214528\", namespace=\"ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-r2hbm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-14T10:09:45Z\n  Ended:       2026-10-14T10:36:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j1691264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.235:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1691264/providers/microsoft.monitor/accounts/hcps-j1691264\", namespace=\"ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-9qldk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-14T10:09:45Z\n  Ended:       2026-10-14T10:36:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j1691264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.235:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1691264/providers/microsoft.monitor/accounts/hcps-j1691264\", namespace=\"ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-9qldk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7301/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110767880390840320","occurred_at":"2026-10-15T16:20:40Z","signature_id":"d8fd96e75384e04fb6b2ce234c1cbb0d72a046901c91b523e45c7d4b95748775","pr_number":7301,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110494690280214528","occurred_at":"2026-10-14T22:15:16Z","signature_id":"30bbfbad94a3e968ca9a9fdf6b8a8083382f6eb5df9064ba2d12e6b538f69cff","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110291728471691264","occurred_at":"2026-10-14T08:48:48Z","signature_id":"02ab642b00e1ec0d87eac69fdef749538d89f3c679b714b14c0e98e0b084fe7c","pr_number":7500,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110131584773394432","occurred_at":"2026-10-13T22:12:28Z","signature_id":"a9ea08e80c10006881b8dcdd8e6fcea6f11b3f2c9b0ce3779517a03365f66b46","pr_number":7509,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"e9d0a6cb09c8c2cfd53af2a6634e712d86d6ad07e23aa28dffb4ebf4cfd08256","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110027575299411968","occurred_at":"2026-10-13T15:15:38Z","signature_id":"812fdff806dd337434b39662e19b80b3cf432a7b1671571c261b606d2d5fc679","pr_number":7485,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109715328622661632","occurred_at":"2026-10-12T18:38:25Z","signature_id":"01b14f16705c690af3cc17499286b7047bac4716208b40b6f4d834caa2ce6ebc","pr_number":7479,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"59230409e21419e206546b3eed7180a2b2d9241870b6db3da89dea2764805770","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","signature_id":"cfbb3abf844cd53f28b64e3663605f410242ad10ab7e9f569183150881111e11","pr_number":7463,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1cd304b95c756f8a1f8ad9e9b7d8205b8dcc2ba19e257386e89c7f15d0fddc01","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":1.629327902240326,"anchor_occurrences":8,"after_last_push_count":3,"trend_counts":[0,2,1,0,4,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":8,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1583fe428d80\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qxkjhfl8b89k\" to finish creating: GET https://rp.j0669824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb248cdb-708c-4e1a-a191-a9d66edb0204\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb248cdb-708c-4e1a-a191-a9d66edb0204\",\n      \"name\": \"fb248cdb-708c-4e1a-a191-a9d66edb0204\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T06:10:02.689570443Z\",\n      \"endTime\": \"2026-10-16T06:29:04.095727153Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng to exist\\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng to exist\\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0669824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb248cdb-708c-4e1a-a191-a9d66edb0204\n            correlation ID: 22bcc657-19c6-431f-9432-afe9f54b1a18\n            client request ID: 0f3d0f1f-4c8a-4108-a1df-7f5cda2678a1\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1583fe428d80\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qxkjhfl8b89k\" to finish creating: GET https://rp.j0669824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb248cdb-708c-4e1a-a191-a9d66edb0204\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb248cdb-708c-4e1a-a191-a9d66edb0204\",\n      \"name\": \"fb248cdb-708c-4e1a-a191-a9d66edb0204\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T06:10:02.689570443Z\",\n      \"endTime\": \"2026-10-16T06:29:04.095727153Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng to exist\\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng to exist\\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0669824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb248cdb-708c-4e1a-a191-a9d66edb0204\n            correlation ID: 22bcc657-19c6-431f-9432-afe9f54b1a18\n            client request ID: 0f3d0f1f-4c8a-4108-a1df-7f5cda2678a1\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d528da08240\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-555prpnqd6q9\" to finish creating: GET https://rp.j4481792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68716f1-756f-4cdd-a508-f1a64147dcc7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68716f1-756f-4cdd-a508-f1a64147dcc7\",\n      \"name\": \"e68716f1-756f-4cdd-a508-f1a64147dcc7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T22:00:18.084367751Z\",\n      \"endTime\": \"2026-10-15T22:29:20.241927727Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-9l6x8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-555prpnqd6q9--managed/providers/Microsoft.Compute/virtualMachines/u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-9l6x8 to exist\\nMachine u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-tcn2v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-555prpnqd6q9--managed/providers/Microsoft.Compute/virtualMachines/u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-tcn2v to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-9l6x8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-555prpnqd6q9--managed/providers/Microsoft.Compute/virtualMachines/u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-9l6x8 to exist\\nMachine u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-tcn2v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-555prpnqd6q9--managed/providers/Microsoft.Compute/virtualMachines/u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-tcn2v to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4481792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68716f1-756f-4cdd-a508-f1a64147dcc7\n            correlation ID: 7e1b7dd4-0022-48fb-91be-ab56afbfcaf9\n            client request ID: eaae09a9-3401-4404-aa69-a9f272cf4e2d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d528da08240\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-555prpnqd6q9\" to finish creating: GET https://rp.j4481792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68716f1-756f-4cdd-a508-f1a64147dcc7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68716f1-756f-4cdd-a508-f1a64147dcc7\",\n      \"name\": \"e68716f1-756f-4cdd-a508-f1a64147dcc7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T22:00:18.084367751Z\",\n      \"endTime\": \"2026-10-15T22:29:20.241927727Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-9l6x8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-555prpnqd6q9--managed/providers/Microsoft.Compute/virtualMachines/u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-9l6x8 to exist\\nMachine u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-tcn2v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-555prpnqd6q9--managed/providers/Microsoft.Compute/virtualMachines/u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-tcn2v to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-9l6x8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-555prpnqd6q9--managed/providers/Microsoft.Compute/virtualMachines/u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-9l6x8 to exist\\nMachine u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-tcn2v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-555prpnqd6q9--managed/providers/Microsoft.Compute/virtualMachines/u5w8y3l2l4x1v4y-nodepool-128gib-pq9vx-tcn2v to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4481792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68716f1-756f-4cdd-a508-f1a64147dcc7\n            correlation ID: 7e1b7dd4-0022-48fb-91be-ab56afbfcaf9\n            client request ID: eaae09a9-3401-4404-aa69-a9f272cf4e2d\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n    \u003c*fmt.wrapError | 0x371fb75e40\u003e: \n    failed to create NodePool swift-workers: failed waiting for nodepool=\"swift-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-htlwf6tksvkk\" to finish creating: GET https://rp.j2353536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b439425f-1c8a-4eb5-8449-8ffb64a115d8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b439425f-1c8a-4eb5-8449-8ffb64a115d8\",\n      \"name\": \"b439425f-1c8a-4eb5-8449-8ffb64a115d8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T21:32:41.543837723Z\",\n      \"endTime\": \"2026-10-15T22:01:41.868424358Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine f0h5o6h1r1j7z7l-swift-workers-wnvr8-4x9jq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-htlwf6tksvkk--managed/providers/Microsoft.Compute/virtualMachines/f0h5o6h1r1j7z7l-swift-workers-wnvr8-4x9jq to exist\\nMachine f0h5o6h1r1j7z7l-swift-workers-wnvr8-t8p9t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-htlwf6tksvkk--managed/providers/Microsoft.Compute/virtualMachines/f0h5o6h1r1j7z7l-swift-workers-wnvr8-t8p9t to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine f0h5o6h1r1j7z7l-swift-workers-wnvr8-4x9jq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-htlwf6tksvkk--managed/providers/Microsoft.Compute/virtualMachines/f0h5o6h1r1j7z7l-swift-workers-wnvr8-4x9jq to exist\\nMachine f0h5o6h1r1j7z7l-swift-workers-wnvr8-t8p9t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-htlwf6tksvkk--managed/providers/Microsoft.Compute/virtualMachines/f0h5o6h1r1j7z7l-swift-workers-wnvr8-t8p9t to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2353536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b439425f-1c8a-4eb5-8449-8ffb64a115d8\n            correlation ID: 11b13ff0-4dd4-474f-81b1-b4341cbc7ec0\n            client request ID: 9a03c5e0-eb37-442c-b9ce-57bebab70a3c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n    \u003c*fmt.wrapError | 0x371fb75e40\u003e: \n    failed to create NodePool swift-workers: failed waiting for nodepool=\"swift-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-htlwf6tksvkk\" to finish creating: GET https://rp.j2353536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b439425f-1c8a-4eb5-8449-8ffb64a115d8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b439425f-1c8a-4eb5-8449-8ffb64a115d8\",\n      \"name\": \"b439425f-1c8a-4eb5-8449-8ffb64a115d8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T21:32:41.543837723Z\",\n      \"endTime\": \"2026-10-15T22:01:41.868424358Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine f0h5o6h1r1j7z7l-swift-workers-wnvr8-4x9jq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-htlwf6tksvkk--managed/providers/Microsoft.Compute/virtualMachines/f0h5o6h1r1j7z7l-swift-workers-wnvr8-4x9jq to exist\\nMachine f0h5o6h1r1j7z7l-swift-workers-wnvr8-t8p9t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-htlwf6tksvkk--managed/providers/Microsoft.Compute/virtualMachines/f0h5o6h1r1j7z7l-swift-workers-wnvr8-t8p9t to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine f0h5o6h1r1j7z7l-swift-workers-wnvr8-4x9jq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-htlwf6tksvkk--managed/providers/Microsoft.Compute/virtualMachines/f0h5o6h1r1j7z7l-swift-workers-wnvr8-4x9jq to exist\\nMachine f0h5o6h1r1j7z7l-swift-workers-wnvr8-t8p9t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-htlwf6tksvkk--managed/providers/Microsoft.Compute/virtualMachines/f0h5o6h1r1j7z7l-swift-workers-wnvr8-t8p9t to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2353536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b439425f-1c8a-4eb5-8449-8ffb64a115d8\n            correlation ID: 11b13ff0-4dd4-474f-81b1-b4341cbc7ec0\n            client request ID: 9a03c5e0-eb37-442c-b9ce-57bebab70a3c\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7564/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110960931310669824","occurred_at":"2026-10-16T05:08:00Z","signature_id":"273e478ae270b5bcb7f9c0960c08545dbd11753951d5c582aa22856eef08182c","pr_number":7564,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110837905944481792","occurred_at":"2026-10-15T20:59:08Z","signature_id":"f93bb3d601671375f8a4e1995e53b2d4dfd2d40ede00eb7e098b8aa8ccfd9790","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110825000012353536","occurred_at":"2026-10-15T20:07:51Z","signature_id":"bcb1583185cccedf8c14d012f708f48aaaed9860131965bc29d2696aeb0299d3","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110780709328130048","occurred_at":"2026-10-15T17:11:52Z","signature_id":"7184f6acc5cd370f6d372fec6df847d10fad413bb134c476a2ee023baed58b21","pr_number":7502,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7436/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110699886142820352","occurred_at":"2026-10-15T11:50:33Z","signature_id":"33d50ea469203d6a26313b26a952123d0a8fb4bfb82a0405de15f90c58f80ade","pr_number":7436,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110035842520059904","occurred_at":"2026-10-13T15:52:02Z","signature_id":"48a02a7f69908511527f6ebb16b479024cc99ba313ab90e8e3951306e211831f","pr_number":7313,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109660256270290944","occurred_at":"2026-10-12T14:59:35Z","signature_id":"222aacc360be62a53a3a25918371654d8976e4d02348bfbb0f85f02afd94062c","pr_number":7373,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109460696247832576","occurred_at":"2026-10-12T01:46:36Z","signature_id":"8363386ebe99557c4d0a5f2ab5fc61129ce9cc8c82392c35b5213d30f26dde65","pr_number":7424,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":1.629327902240326,"anchor_occurrences":8,"after_last_push_count":6,"trend_counts":[0,2,0,2,4,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-10-05"],"prior_runs_affected":18,"prior_last_seen_at":"2026-10-09T07:25:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x2d72cc8aa180\u003e: \n    resource group \"autoscaling-cluster-7hjmhzg78tkl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-7hjmhzg78tkl\" to finish deleting: GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c3202962-b736-4f51-840b-2c3b32fd9172\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c3202962-b736-4f51-840b-2c3b32fd9172\",\n      \"name\": \"c3202962-b736-4f51-840b-2c3b32fd9172\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T18:00:50.353777Z\",\n      \"endTime\": \"2026-10-15T18:24:53.714360344Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tfbvc1dj03nufhiu0qumf4o5mmkbqp1 still exists (deletion dispatched at 2026-10-15T18:00:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c3202962-b736-4f51-840b-2c3b32fd9172\n            correlation ID: 2ab924cd-c6a5-4535-afa1-4285e0d7421a\n            client request ID: 05581b93-1450-43ab-a2a8-826d6a1faeda\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x2d72cc8aa180\u003e: \n    resource group \"autoscaling-cluster-7hjmhzg78tkl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-7hjmhzg78tkl\" to finish deleting: GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c3202962-b736-4f51-840b-2c3b32fd9172\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c3202962-b736-4f51-840b-2c3b32fd9172\",\n      \"name\": \"c3202962-b736-4f51-840b-2c3b32fd9172\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T18:00:50.353777Z\",\n      \"endTime\": \"2026-10-15T18:24:53.714360344Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tfbvc1dj03nufhiu0qumf4o5mmkbqp1 still exists (deletion dispatched at 2026-10-15T18:00:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c3202962-b736-4f51-840b-2c3b32fd9172\n            correlation ID: 2ab924cd-c6a5-4535-afa1-4285e0d7421a\n            client request ID: 05581b93-1450-43ab-a2a8-826d6a1faeda\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0xb39be402570\u003e: \n    resource group \"rg-zstream-upgrade-4-22-qkz2pwtj5n7j\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-9x7njw\" in resourcegroup=\"rg-zstream-upgrade-4-22-qkz2pwtj5n7j\" to finish deleting: GET https://rp.j0325120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d1c8a5-60e3-4c19-acaf-41b8b35cb86a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d1c8a5-60e3-4c19-acaf-41b8b35cb86a\",\n      \"name\": \"a5d1c8a5-60e3-4c19-acaf-41b8b35cb86a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T17:05:09.736854856Z\",\n      \"endTime\": \"2026-10-15T17:29:18.684928477Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tfb63s18q5cesobi99a7b7bb7k3a72n still exists (deletion dispatched at 2026-10-15T17:05:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0325120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d1c8a5-60e3-4c19-acaf-41b8b35cb86a\n            correlation ID: ce6f6715-7b02-4412-b5e3-4934b1a27812\n            client request ID: bf6cd98a-47f4-43d5-af65-2ee26c5ddfb6\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0xb39be402570\u003e: \n    resource group \"rg-zstream-upgrade-4-22-qkz2pwtj5n7j\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-9x7njw\" in resourcegroup=\"rg-zstream-upgrade-4-22-qkz2pwtj5n7j\" to finish deleting: GET https://rp.j0325120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d1c8a5-60e3-4c19-acaf-41b8b35cb86a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d1c8a5-60e3-4c19-acaf-41b8b35cb86a\",\n      \"name\": \"a5d1c8a5-60e3-4c19-acaf-41b8b35cb86a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T17:05:09.736854856Z\",\n      \"endTime\": \"2026-10-15T17:29:18.684928477Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tfb63s18q5cesobi99a7b7bb7k3a72n still exists (deletion dispatched at 2026-10-15T17:05:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0325120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d1c8a5-60e3-4c19-acaf-41b8b35cb86a\n            correlation ID: ce6f6715-7b02-4412-b5e3-4934b1a27812\n            client request ID: bf6cd98a-47f4-43d5-af65-2ee26c5ddfb6\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0xa89f6d94528\u003e: \n    resource group \"rg-np-noedge-tw7cjj-xlndscvj7dwp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-tw7cjj\" in resourcegroup=\"rg-np-noedge-tw7cjj-xlndscvj7dwp\" to finish deleting: GET https://rp.j5616384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/19704973-55a6-47ab-b561-11bb7cb66fb8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/19704973-55a6-47ab-b561-11bb7cb66fb8\",\n      \"name\": \"19704973-55a6-47ab-b561-11bb7cb66fb8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T14:51:30.063016611Z\",\n      \"endTime\": \"2026-10-15T15:15:36.942553753Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tf9cp7cug191vvs2krpcsqb89jucjnj still exists (deletion dispatched at 2026-10-15T14:51:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5616384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/19704973-55a6-47ab-b561-11bb7cb66fb8\n            correlation ID: 6de0dbef-67fc-48b2-ad3a-a51b2af3faa1\n            client request ID: a410cfe8-06c4-4eb7-afde-a18a2ead7f72\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0xa89f6d94528\u003e: \n    resource group \"rg-np-noedge-tw7cjj-xlndscvj7dwp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-tw7cjj\" in resourcegroup=\"rg-np-noedge-tw7cjj-xlndscvj7dwp\" to finish deleting: GET https://rp.j5616384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/19704973-55a6-47ab-b561-11bb7cb66fb8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/19704973-55a6-47ab-b561-11bb7cb66fb8\",\n      \"name\": \"19704973-55a6-47ab-b561-11bb7cb66fb8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T14:51:30.063016611Z\",\n      \"endTime\": \"2026-10-15T15:15:36.942553753Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tf9cp7cug191vvs2krpcsqb89jucjnj still exists (deletion dispatched at 2026-10-15T14:51:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5616384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/19704973-55a6-47ab-b561-11bb7cb66fb8\n            correlation ID: 6de0dbef-67fc-48b2-ad3a-a51b2af3faa1\n            client request ID: a410cfe8-06c4-4eb7-afde-a18a2ead7f72\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110772387883520000","occurred_at":"2026-10-15T16:38:43Z","signature_id":"7aeb2f131c05861e354a5058aad84582e5e6176ebb7c265fa559e5539a40f301","pr_number":7560,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110758849320325120","occurred_at":"2026-10-15T15:45:00Z","signature_id":"e97587f9f1752a822ac9a2d6c925afe1277cba6a9e2cb035a9dd9e3d406575ef","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110717603545616384","occurred_at":"2026-10-15T13:00:33Z","signature_id":"388ba27c7c574e60f2fa23b9d3df6b309e0483c1e693065b5d4862ee58e4493c","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110607466327707648","occurred_at":"2026-10-15T05:43:27Z","signature_id":"b7fa26eab1a720d5e3645d12c57cf4f2ad36521990ec6cafd9cf46aa09d22a60","pr_number":7509,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7301/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110412681776205824","occurred_at":"2026-10-14T16:49:27Z","signature_id":"805ce9687446a53482c8cbf283a80c102b3c36ffbdef6a91f5aa6520b895724e","pr_number":7301,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"1c56e7df692c6920b8c980b71c8374582ba0ef9f87b43803546d8d8bbfe1ae3b","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"f1b1b884321d5b2ac6d5d82d2bb343fddd2a8c66964ab52673d16c25eb8d5697","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109600459105767424","occurred_at":"2026-10-12T11:01:58Z","signature_id":"5b41a952bcf6d2d929fc1668bfb207665a25d88b8be5793b80fd4bb0cf28e575","pr_number":7468,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":1.629327902240326,"anchor_occurrences":8,"after_last_push_count":7,"trend_counts":[0,0,1,2,3,2,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-10-05"],"prior_runs_affected":21,"prior_last_seen_at":"2026-10-09T14:02:38Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":8}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-16T15:54:14Z\n  Ended:       2026-10-16T16:02:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j7660416-mgmt-1\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-16T15:55:11Z\n  Ended:       2026-10-16T16:02:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j7660416-mgmt-2\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T14:26:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j8714496-mgmt-2\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T17:43:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j4658048-mgmt-2\", component=\"monitoring-infrastructure\", region=\"centralus\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7512/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111107720617660416","occurred_at":"2026-10-16T14:51:17Z","signature_id":"26ad01559a977d2ccbf9de49f0e0a12ad85e844d43c5c076830f46ba6268215f","pr_number":7512,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111087082628714496","occurred_at":"2026-10-16T13:29:17Z","signature_id":"4f5e43f8961ce1450724da4818b03afc4ea4a1e65b095d58ea6cf9018f4c082d","pr_number":7232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110779912934658048","occurred_at":"2026-10-15T17:08:42Z","signature_id":"050f4c440f64150e7970ac13f573a27b5f50a5f05fdeb18033f081e39ff717b6","pr_number":7509,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110762016095342592","occurred_at":"2026-10-15T15:57:35Z","signature_id":"22ea0c3803bb98326df9cbf23de00a80472321a99d7d574f14fa44cb6663b005","pr_number":6746,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110545325142839296","occurred_at":"2026-10-15T01:36:32Z","signature_id":"050f4c440f64150e7970ac13f573a27b5f50a5f05fdeb18033f081e39ff717b6","pr_number":7133,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6946/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110441834562981888","occurred_at":"2026-10-14T18:45:15Z","signature_id":"4f5e43f8961ce1450724da4818b03afc4ea4a1e65b095d58ea6cf9018f4c082d","pr_number":6946,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110398398338174976","occurred_at":"2026-10-14T15:52:39Z","signature_id":"b44e1602655f10f93a02f5aad2f805d9843cc1136bdf9380aaa0cc48e7872d35","pr_number":7509,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110042774484029440","occurred_at":"2026-10-13T16:19:32Z","signature_id":"26ad01559a977d2ccbf9de49f0e0a12ad85e844d43c5c076830f46ba6268215f","pr_number":7502,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":1.4256619144602851,"anchor_occurrences":7,"after_last_push_count":6,"trend_counts":[0,0,0,5,1,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":32,"prior_last_seen_at":"2026-10-09T21:48:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x8a369199c00\u003e: \n    DNS for route host agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x8a36a1383e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x8a369199c00\u003e: \n    DNS for route host agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x8a36a1383e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-5bxn7w\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x37da2d0209a0\u003e: \n    DNS for route host agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x37da2d0168c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-5bxn7w\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x37da2d0209a0\u003e: \n    DNS for route host agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x37da2d0168c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-v5x65.apps.aro.j6r3d2d0n3v4o5x.zbh9.j2571392.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1fd36b60420\u003e: \n    DNS for route host agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1fd36b60400\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1fd36b60420\u003e: \n    DNS for route host agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1fd36b60400\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110993516539678720","occurred_at":"2026-10-16T07:17:29Z","signature_id":"67d8158e7a98b23599a95a7df825525e560fd538e129d6bd8ff14bb6c6993c02","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7382/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110810466912571392","occurred_at":"2026-10-15T19:10:04Z","signature_id":"ec51b718fd0d45166ba23428136b5566271d1ccb25c871137506b1dd798ddffa","pr_number":7382,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110371366967447552","occurred_at":"2026-10-14T14:05:09Z","signature_id":"9e22c62935cd79e964f918ea959f94f3511d0bddaffe8d45ff361c9c1abb7ca7","pr_number":7519,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7430/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110316641895059456","occurred_at":"2026-10-14T10:27:49Z","signature_id":"ed1ed5ed2432b0c9c13a6ec98fa3bb36db707b1b3b979c88ae7437a7a6e87e63","pr_number":7430,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110275068176109568","occurred_at":"2026-10-14T07:42:37Z","signature_id":"815c2d271041727d3fd2c78730b42d4a5396d584aa19c429bcd82b945077a214","pr_number":7492,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7515/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110253677582225408","occurred_at":"2026-10-14T06:17:37Z","signature_id":"f9e54dc2e97245d91f304a9f6c2ad9e348ba515da09f1b844adfe4d35dd65d2f","pr_number":7515,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110168566228586496","occurred_at":"2026-10-14T00:39:25Z","signature_id":"ad240022cb81ab8363b3db81cf92564f82d7d9330d100781ceec147c73181eea","pr_number":7248,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":1.4256619144602851,"anchor_occurrences":7,"after_last_push_count":13,"trend_counts":[0,2,1,1,2,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":63,"prior_last_seen_at":"2026-10-09T14:02:38Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":7}],"full_error_samples":["alert fired 18 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:03:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-c4d6b786-548pt\", region=\"westus3\", severity=\"warning\"\n  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-c4d6b786-548pt has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:05:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-5lt2p\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/aks-secrets-store-csi-driver-5lt2p has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:05:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-nw2d2\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/aks-secrets-store-csi-driver-nw2d2 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:05:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-rsnz7\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/aks-secrets-store-csi-driver-rsnz7 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:05:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-q6lgt\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/ama-metrics-node-q6lgt has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:05:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-27216613-vmss000000-jg5jx\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/eraser-aks-system-27216613-vmss000000-jg5jx has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:05:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"hubble-relay-58594b5dd7-c85j5\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/hubble-relay-58594b5dd7-c85j5 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:05:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-9j44l\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/ama-metrics-node-9j44l has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:05:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-27216613-vmss000002-wgh5j\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/eraser-aks-system-27216613-vmss000002-wgh5j has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:05:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-27216613-vmss000001-zvc5c\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/eraser-aks-system-27216613-vmss000001-zvc5c has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:05:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-bjvb8\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/ama-metrics-node-bjvb8 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:13:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-6cb67d7f44-f4tbd\", region=\"westus3\", severity=\"warning\"\n  Description: Pod clusters-service/clusters-service-6cb67d7f44-f4tbd has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:13:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"sessiongate\", pod=\"sessiongate-77fbb4f5f5-2fjpq\", region=\"westus3\", severity=\"warning\"\n  Description: Pod sessiongate/sessiongate-77fbb4f5f5-2fjpq has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:13:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-6cb67d7f44-89fpd\", region=\"westus3\", severity=\"warning\"\n  Description: Pod clusters-service/clusters-service-6cb67d7f44-89fpd has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:13:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-6cb67d7f44-g6qp6\", region=\"westus3\", severity=\"warning\"\n  Description: Pod clusters-service/clusters-service-6cb67d7f44-g6qp6 has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:13:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"maestro\", pod=\"maestro-7dfb5b7d6b-dgw7q\", region=\"westus3\", severity=\"warning\"\n  Description: Pod maestro/maestro-7dfb5b7d6b-dgw7q has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:13:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"maestro\", pod=\"maestro-7dfb5b7d6b-qws2j\", region=\"westus3\", severity=\"warning\"\n  Description: Pod maestro/maestro-7dfb5b7d6b-qws2j has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:13:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"sessiongate\", pod=\"sessiongate-77fbb4f5f5-gdqfm\", region=\"westus3\", severity=\"warning\"\n  Description: Pod sessiongate/sessiongate-77fbb4f5f5-gdqfm has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T20:18:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5701632-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-c4d6b786-v5xlx\", region=\"centralus\", severity=\"warning\"\n  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-c4d6b786-v5xlx has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T17:40:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4658048-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"set-kubelet-parameters-for-scale-r7jxg\", region=\"centralus\", severity=\"warning\"\n  Description: Pod kube-system/set-kubelet-parameters-for-scale-r7jxg has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7512/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111107720617660416","occurred_at":"2026-10-16T14:51:17Z","signature_id":"d07de28f589089258ca9dc6edac09be43e6962d6a3ef23500a1585d2a82b650c","pr_number":7512,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110810477125701632","occurred_at":"2026-10-15T19:10:09Z","signature_id":"74f21da1deaee926f942e6526f8c93b2c43ab3fcc9db5dea46cefe3240a6a7e4","pr_number":7560,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110779912934658048","occurred_at":"2026-10-15T17:08:42Z","signature_id":"169aadbe10dd9676b1fa8c8f019026f3191971176344663d51fcbf69d723cf02","pr_number":7509,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110417722436227072","occurred_at":"2026-10-14T17:09:29Z","signature_id":"7a7c5e86b9ade1b9e5367692230bf2de020f056cf10152974063cca4acfddab3","pr_number":7519,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110042774484029440","occurred_at":"2026-10-13T16:19:32Z","signature_id":"761776c84216b3b60fba1a6f060188d98d184222bdf626e7b5bac2005be4c5d7","pr_number":7502,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109581128020004864","occurred_at":"2026-10-12T09:45:06Z","signature_id":"6e8b49a5caf85ad1a9ceb24783ca3a6237c65bb0981b2afa932bd94b35ce4433","pr_number":7439,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109578665191804928","occurred_at":"2026-10-12T09:35:22Z","signature_id":"b224cee0c66252329215d260009da282cdb5e711becf7fce204c7d1cc07cca6f","pr_number":7162,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bb4de136ebd0024068fdae71f01ccc40217c2ff8bba4e521e07b515791de06dd","failure_pattern":"cluster \"\u003ccluster\u003e\" did not complete the SRE-pinned version 4.21.36","search_query":"cluster \"sre-version-pin-5gfztc\" did not complete the SRE-pinned version 4.21.36","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE advances a control plane to an SRE-pinned version","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":1.4256619144602851,"anchor_occurrences":7,"after_last_push_count":0,"trend_counts":[0,5,2,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE advances a control plane to an SRE-pinned version","occurrences":7}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.005s.\ncluster \"sre-version-pin-z4tqzw\" did not complete the SRE-pinned version 4.21.36\nExpected\n    \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n    \u003cstring\u003e: 4.21.36\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.005s.\ncluster \"sre-version-pin-z4tqzw\" did not complete the SRE-pinned version 4.21.36\nExpected\n    \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n    \u003cstring\u003e: 4.21.36","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.002s.\ncluster \"sre-version-pin-sjbwq2\" did not complete the SRE-pinned version 4.21.36\nExpected\n    \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n    \u003cstring\u003e: 4.21.36\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.002s.\ncluster \"sre-version-pin-sjbwq2\" did not complete the SRE-pinned version 4.21.36\nExpected\n    \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n    \u003cstring\u003e: 4.21.36","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.003s.\ncluster \"sre-version-pin-k9v4gs\" did not complete the SRE-pinned version 4.21.36\nExpected\n    \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n    \u003cstring\u003e: 4.21.36\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.003s.\ncluster \"sre-version-pin-k9v4gs\" did not complete the SRE-pinned version 4.21.36\nExpected\n    \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n    \u003cstring\u003e: 4.21.36"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109934998684438528","occurred_at":"2026-10-13T09:11:18Z","signature_id":"e726ff68ce5087e7b46536a414b0c4ce318e72dc425aac96edae158603185df3","pr_number":7383,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109798012107624448","occurred_at":"2026-10-13T00:06:58Z","signature_id":"17cac32c9f196891eb1cddda2b22294095ac725cb2ad07869f1b1d0d4e898ccd","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109754183480512512","occurred_at":"2026-10-12T21:12:49Z","signature_id":"8ec28f71b625b2c11ea506fd0748031d3bf5d074d3279546ef6bfc6cf0d20aaf","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109675831700754432","occurred_at":"2026-10-12T16:01:28Z","signature_id":"22c47845096418cb71e4fa35973233fa4b21f339d382954b3aee25b82c29cb84","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109662114334380032","occurred_at":"2026-10-12T15:06:56Z","signature_id":"74adfdd35c7ce8eb88d441864d6b1e0ec991a2d255378a9ffce5b6cd5cf8549e","pr_number":7383,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109629712874606592","occurred_at":"2026-10-12T12:58:13Z","signature_id":"b12b641e87320622b3d715665e73509b0e2928b9b8bc71c938cb2b7200e48d3e","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109600098458537984","occurred_at":"2026-10-12T11:00:15Z","signature_id":"f0121469ca4f42ea3b4cc9ff22e2d69e6a7cdd34679fb94a79eb413022a0a715","pr_number":7383,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4275a314d89bf06a724f7bf3c5bfe06406a90fa00482d282b46f40a1b17ff273","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j0532352 does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":1.2219959266802445,"anchor_occurrences":6,"after_last_push_count":0,"trend_counts":[0,1,4,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j0532352 does not fire","occurrences":1},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j2816128 does not fire","occurrences":1},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j6775168 does not fire","occurrences":1},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7369856 does not fire","occurrences":1},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7684864 does not fire","occurrences":1},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j9626368 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-14T03:48:00Z\n  Ended:       2026-10-14T04:18:05Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 15-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-14T00:46:13Z\n  Ended:       2026-10-14T01:18:14Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 15-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-13T22:09:34Z\n  Ended:       2026-10-13T22:42:14Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 15-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110199338117369856","occurred_at":"2026-10-14T02:41:42Z","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110153446450532352","occurred_at":"2026-10-13T23:39:20Z","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110115702512816128","occurred_at":"2026-10-13T21:09:22Z","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110080447856775168","occurred_at":"2026-10-13T17:40:39Z","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110029189787684864","occurred_at":"2026-10-13T15:25:35Z","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":1.2219959266802445,"anchor_occurrences":6,"after_last_push_count":7,"trend_counts":[0,0,2,1,2,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":21,"prior_last_seen_at":"2026-10-08T12:00:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:365]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x26aec5b792f0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40106-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40106-\u003e172.30.0.10:53: read: connection refused\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:365]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x26aec5b792f0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40106-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40106-\u003e172.30.0.10:53: read: connection refused\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:365]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2b9d750c1520\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58152-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58152-\u003e172.30.0.10:53: read: connection refused\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:365]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2b9d750c1520\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58152-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58152-\u003e172.30.0.10:53: read: connection refused\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:365]: Timed out after 300.006s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3f466db88d40\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:33148-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:33148-\u003e172.30.0.10:53: read: connection refused\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:365]: Timed out after 300.006s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3f466db88d40\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:33148-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:33148-\u003e172.30.0.10:53: read: connection refused\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7539/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111076959755702272","occurred_at":"2026-10-16T12:49:03Z","signature_id":"e62ea6a200fd389ea519e04ea59463bd4151415eaccd997a4c214c2e3763a28a","pr_number":7539,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110728083584585728","occurred_at":"2026-10-15T13:42:45Z","signature_id":"a4935669284b57675a96275c47c46b1a5544bb65fce6d2bb8f50bf15a0d24e7a","pr_number":7387,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110633585886105600","occurred_at":"2026-10-15T07:27:14Z","signature_id":"84817028137321b246b8f3fba13ef23f0d9ed8a4d231cdc3b62618bb815ca186","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110483335657558016","occurred_at":"2026-10-14T21:30:12Z","signature_id":"87f77c6532d69c77561fa791550c191175a258371f754dfc003a6b27e5e2704d","pr_number":7524,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110141907093950464","occurred_at":"2026-10-13T22:53:29Z","signature_id":"b866590f9b46bfea6312f2e0fb04cbc1e34def7eb9a707daec346d608f89ac23","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","signature_id":"7728ea8ada183ffb061ff6026d3b0066e425d012b33ecc69f14c383fb512405c","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fbdf4b29275bbcfcba0aba86a244253ee3cdcaa74712c6d2e008d1ca9e4f19bc","failure_pattern":"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \"globalMSIId\" not found","search_query":"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \\\"globalMSIId\\\" not found","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":1.2219959266802445,"anchor_occurrences":6,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,4,0,2],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":6}],"full_error_samples":["time=2026-10-18T15:38:05.575Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-18T15:38:06.914Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \\\"globalMSIId\\\" not found\"","time=2026-10-18T12:10:56.819Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-18T12:10:57.618Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \\\"globalMSIId\\\" not found\"","time=2026-10-16T23:16:01.424Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-16T23:16:02.804Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \\\"globalMSIId\\\" not found\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111837676591124480","occurred_at":"2026-10-18T15:11:52Z","signature_id":"6c85c0c84a029cc468c08af59c9556ea1e898db311553fdd2c177b18ce147d44","pr_number":7580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111776757118930944","occurred_at":"2026-10-18T11:09:48Z","signature_id":"6c85c0c84a029cc468c08af59c9556ea1e898db311553fdd2c177b18ce147d44","pr_number":7580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111233385828454400","occurred_at":"2026-10-16T23:10:38Z","signature_id":"6c85c0c84a029cc468c08af59c9556ea1e898db311553fdd2c177b18ce147d44","pr_number":7580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111219106693779456","occurred_at":"2026-10-16T22:13:54Z","signature_id":"6c85c0c84a029cc468c08af59c9556ea1e898db311553fdd2c177b18ce147d44","pr_number":7580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111146056094846976","occurred_at":"2026-10-16T17:23:37Z","signature_id":"6c85c0c84a029cc468c08af59c9556ea1e898db311553fdd2c177b18ce147d44","pr_number":7580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111136882900340736","occurred_at":"2026-10-16T16:47:10Z","signature_id":"6c85c0c84a029cc468c08af59c9556ea1e898db311553fdd2c177b18ce147d44","pr_number":7580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert collection is complete","occurrences":6,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.6109979633401222,"anchor_occurrences":6,"after_last_push_count":8,"trend_counts":[0,4,2,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-09T09:35:37Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert collection is complete","occurrences":3},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert collection is complete","occurrences":3}],"full_error_samples":["alert collection incomplete\nfailed to fetch hcp alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"ResourceGroupNotFound\",\n    \"message\": \"Resource group 'hcp-underlay-ci01-usw3' could not be found.\"\n  }\n}\n--------------------------------------------------------------------------------","alert collection incomplete\nfailed to fetch svc alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"ResourceGroupNotFound\",\n    \"message\": \"Resource group 'hcp-underlay-ci01-usw3' could not be found.\"\n  }\n}\n--------------------------------------------------------------------------------","alert collection incomplete\nfailed to fetch hcp alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9114496/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"ResourceGroupNotFound\",\n    \"message\": \"Resource group 'hcp-underlay-ci01-j9114496' could not be found.\"\n  }\n}\n--------------------------------------------------------------------------------"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7501/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110030920055525376","occurred_at":"2026-10-13T15:32:24Z","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","pr_number":7501,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7501/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110030920055525376","occurred_at":"2026-10-13T15:32:24Z","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","pr_number":7501,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109792838379114496","occurred_at":"2026-10-12T23:46:25Z","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","pr_number":7439,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109792838379114496","occurred_at":"2026-10-12T23:46:25Z","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","pr_number":7439,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109727639982837760","occurred_at":"2026-10-12T19:27:20Z","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109727639982837760","occurred_at":"2026-10-12T19:27:20Z","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","pr_number":6920,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ca6140fe6832e567dbdbd62e69723ab444b3c85ac16836f37268901fc2e842b0","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Timeout; detail message Operation is being Canceled due to timeout.; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":1.0183299389002036,"anchor_occurrences":5,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,5,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":5}],"full_error_samples":["time=2026-10-16T04:32:52.168Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-16T04:32:53.669Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-10-16T04:32:54.816Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=52b936b326777547bb7821b194e68c26b111cb76ef998bdad29331fe388cab6a portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3854848%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F52b936b326777547bb7821b194e68c26b111cb76ef998bdad29331fe388cab6a\ntime=2026-10-16T04:48:28.496Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3854848/providers/Microsoft.Resources/deployments/52b936b326777547bb7821b194e68c26b111cb76ef998bdad29331fe388cab6a/operationStatuses/08584094817114313637\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3854848/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3854848/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j3854848\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Timeout\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Operation is being Canceled due to timeout.\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-10-16T03:17:16.536Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-16T03:17:18.501Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-10-16T03:17:19.523Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=23386a13ae4b9c10e1167d28d0eb0e67c647056f11a0c2bc948bc314d383ed57 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6526208%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F23386a13ae4b9c10e1167d28d0eb0e67c647056f11a0c2bc948bc314d383ed57\ntime=2026-10-16T03:35:23.849Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6526208/providers/Microsoft.Resources/deployments/23386a13ae4b9c10e1167d28d0eb0e67c647056f11a0c2bc948bc314d383ed57/operationStatuses/08584094862465785031\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6526208/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6526208/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6526208\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Timeout\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Operation is being Canceled due to timeout.\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-10-16T02:17:33.774Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-16T02:17:35.249Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-10-16T02:17:36.184Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=11568e45e85c71193ac85f28e42fe8900978b11d9ee87376f97b514ef517dce8 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5876608%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F11568e45e85c71193ac85f28e42fe8900978b11d9ee87376f97b514ef517dce8\ntime=2026-10-16T02:33:38.292Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5876608/providers/Microsoft.Resources/deployments/11568e45e85c71193ac85f28e42fe8900978b11d9ee87376f97b514ef517dce8/operationStatuses/08584094898298983239\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5876608/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5876608/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j5876608\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Timeout\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Operation is being Canceled due to timeout.\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110946452283854848","occurred_at":"2026-10-16T04:10:28Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":6746,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110923393426526208","occurred_at":"2026-10-16T02:38:50Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":7250,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7561/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110912562055876608","occurred_at":"2026-10-16T01:55:48Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":7561,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110910911651778560","occurred_at":"2026-10-16T01:49:08Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":7250,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110909868377378816","occurred_at":"2026-10-16T01:45:06Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":6746,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2","failure_pattern":"alert [hcp] HCPClusterOperatorUnavailable fired","search_query":"alert [hcp] HCPClusterOperatorUnavailable fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert HCPClusterOperatorUnavailable does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":1.0183299389002036,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,3,0,0,0,2,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":10,"prior_last_seen_at":"2026-10-07T12:59:41Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert HCPClusterOperatorUnavailable does not fire","occurrences":5}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-16T02:38:55Z\n  Ended:       2026-10-16T03:16:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j2423808-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj30j2k6sdjh9r7vsbj6npes2m40m2-p2r6x1t9q6d6f9l\", region=\"centralus\", severity=\"warning\"\n  Description: 1 cluster operator(s) on hosted cluster ocm-arohcpci01-2tfj30j2k6sdjh9r7vsbj6npes2m40m2-p2r6x1t9q6d6f9l (management cluster ci01-j2423808-mgmt-1) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-16T02:40:29Z\n  Ended:       2026-10-16T03:15:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j3528064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3qrd1lfn90p1i58kr795ndsi2u3g-f4z5z8b0a1u0o2v\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2tfj3qrd1lfn90p1i58kr795ndsi2u3g-f4z5z8b0a1u0o2v (management cluster ci01-j3528064-mgmt-2) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T21:30:33Z\n  Ended:       2026-10-12T22:07:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j6043392-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdf8lncsu04rjm64fbreumbi4l47bg3-l5g9b5r3v2p1b0s\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2tdf8lncsu04rjm64fbreumbi4l47bg3-l5g9b5r3v2p1b0s (management cluster ci01-j6043392-mgmt-2) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110896106962423808","occurred_at":"2026-10-16T00:50:25Z","signature_id":"b17a76908c996524ea705a1504c61f6666570717cb26311f2fea20b9e82bf822","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110895015453528064","occurred_at":"2026-10-16T00:46:04Z","signature_id":"5d614e354a186d8a4f21f749eb16466b0722b8015b284c219df8c2b54634da4e","pr_number":7471,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109729919306043392","occurred_at":"2026-10-12T19:36:16Z","signature_id":"0c18256aec4487d73b73645b8e1ded1390ad9218643d5179523cb223b9edd0b5","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109706138382503936","occurred_at":"2026-10-12T18:01:54Z","signature_id":"1e41fec641cef36701893933a54d28e9a3e41519280905104056febcd0e7dcfb","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109705158475649024","occurred_at":"2026-10-12T17:58:00Z","signature_id":"1de0d91a762f7a24003e4b3846ddd93be7eb53f9d5a9baa369ea319d275302ba","pr_number":7471,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-2n7cffz9j9vb\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":1.0183299389002036,"anchor_occurrences":5,"after_last_push_count":1,"trend_counts":[0,1,0,2,1,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":14,"prior_last_seen_at":"2026-10-09T17:15:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x248dbdd6a2d0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-p9pjc5tph49f\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-p9pjc5tph49f, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-p9pjc5tph49f\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-p9pjc5tph49f, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x248dbde747e0\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-p9pjc5tph49f\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x248dbdd6a2d0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-p9pjc5tph49f\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-p9pjc5tph49f, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-p9pjc5tph49f\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-p9pjc5tph49f, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x248dbde747e0\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-p9pjc5tph49f\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x3e6f09e44c00\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-k7gbzwzskr9p\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-k7gbzwzskr9p, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-k7gbzwzskr9p\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-k7gbzwzskr9p, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x3e6f0a152150\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-k7gbzwzskr9p\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x3e6f09e44c00\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-k7gbzwzskr9p\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-k7gbzwzskr9p, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-k7gbzwzskr9p\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-k7gbzwzskr9p, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x3e6f0a152150\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-k7gbzwzskr9p\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x1399efb46de0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-2n7cffz9j9vb\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-2n7cffz9j9vb\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x1399ef809d30\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x1399efb46de0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-2n7cffz9j9vb\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-2n7cffz9j9vb\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x1399ef809d30\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"b93bac5ee55d46ed1d89154858d9b08bd23b63b8774e6e33f1c1c14ab66fca7f","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110655550189473792","occurred_at":"2026-10-15T08:54:31Z","signature_id":"4157c3a50fb0d8169c14569cdd2fc75c8c37a0edc8c9c3b571cf6a22d23f1694","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110436010226618368","occurred_at":"2026-10-14T18:22:06Z","signature_id":"f01d09e5b26ed293c1f2fb9f79dc527c3d66d0ac5cf3f21d86318c9622720514","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"ac0f754d7b800e55b8405f91d30db0dad28eccf8fd68556cf99e019a8b65a48f","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"bec2ded38a9a866918a5ba9ad1e9a064412687841aabfca8c59e91dc8567ad9c","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"56d972e76712743c510249b7bfaf8d1cef7f97d62d33d6258594d264ca2ff6b2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,0,0,5,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-05T23:05:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-8fmvrw\" to 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x189d56536760\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-8fmvrw\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-m4rkshgrgs9x\" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912\",\n      \"name\": \"008de110-0fc7-4cfb-b1a8-16dd774cc912\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T03:16:19.986510176Z\",\n      \"endTime\": \"2026-10-14T04:00:20.242906036Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912\n            correlation ID: edcafa24-51c5-48f1-9ce5-cdc56c9cfe21\n            client request ID: d7ffd994-e6d8-430b-b52c-c25abf00d2d4\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-8fmvrw\" to 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x189d56536760\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-8fmvrw\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-m4rkshgrgs9x\" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912\",\n      \"name\": \"008de110-0fc7-4cfb-b1a8-16dd774cc912\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T03:16:19.986510176Z\",\n      \"endTime\": \"2026-10-14T04:00:20.242906036Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912\n            correlation ID: edcafa24-51c5-48f1-9ce5-cdc56c9cfe21\n            client request ID: d7ffd994-e6d8-430b-b52c-c25abf00d2d4\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-v9gdsd\" to 5.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0x89c8a7b4520\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-v9gdsd\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-lrtt9r887h59\" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0\",\n      \"name\": \"149ad663-a4d9-4ebd-9101-8c188eed65d0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T03:29:27.04612928Z\",\n      \"endTime\": \"2026-10-14T04:13:34.922970159Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0\n            correlation ID: b89bc015-38c4-4b8f-8cf6-fd7d29e794e7\n            client request ID: c975ef93-0023-4a12-b6b6-8dfd3c0b6abf\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-v9gdsd\" to 5.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0x89c8a7b4520\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-v9gdsd\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-lrtt9r887h59\" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0\",\n      \"name\": \"149ad663-a4d9-4ebd-9101-8c188eed65d0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T03:29:27.04612928Z\",\n      \"endTime\": \"2026-10-14T04:13:34.922970159Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0\n            correlation ID: b89bc015-38c4-4b8f-8cf6-fd7d29e794e7\n            client request ID: c975ef93-0023-4a12-b6b6-8dfd3c0b6abf\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-trt4r8\" to 4.23\nUnexpected error:\n    \u003c*fmt.wrapError | 0xbc2497a0a40\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-trt4r8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-nm455m5dz4rg\" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3\",\n      \"name\": \"ab2c099b-e08d-48bf-8615-536073622db3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T03:17:19.784745543Z\",\n      \"endTime\": \"2026-10-14T04:01:20.236478176Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3\n            correlation ID: b6e77c38-1227-46e6-9bc5-a4baba263145\n            client request ID: fcf783cc-37c0-4ef5-8f68-49e5b907b57c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-trt4r8\" to 4.23\nUnexpected error:\n    \u003c*fmt.wrapError | 0xbc2497a0a40\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-trt4r8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-nm455m5dz4rg\" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3\",\n      \"name\": \"ab2c099b-e08d-48bf-8615-536073622db3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T03:17:19.784745543Z\",\n      \"endTime\": \"2026-10-14T04:01:20.236478176Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3\n            correlation ID: b6e77c38-1227-46e6-9bc5-a4baba263145\n            client request ID: fcf783cc-37c0-4ef5-8f68-49e5b907b57c\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110191032388816896","occurred_at":"2026-10-14T02:08:42Z","signature_id":"5d250e50a264778942c74b3ce0da1834bd17192a108f576c77963d2bd5104c98","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110191032388816896","occurred_at":"2026-10-14T02:08:42Z","signature_id":"850a794b118cd701611798b45a5bcb03f88a9401ed189a5b8f264b754f82d23a","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110191032388816896","occurred_at":"2026-10-14T02:08:42Z","signature_id":"8825f796a0bcda6f47b5a66cc92693cc86f0c5ca1b9550e8c9c028e39821e838","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110191032388816896","occurred_at":"2026-10-14T02:08:42Z","signature_id":"93eac86ad49dfa02d26b8b40788a25a3f63d761987ff78fe67ff401dea4135e9","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110191032388816896","occurred_at":"2026-10-14T02:08:42Z","signature_id":"e89a0c8726bcc2e849495d81caf51843d12d66f11e5cc3df7b6a10e2304b7d01","pr_number":6968,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9205c69c72a57356c2890f3b58a265a6f5459948112af87f37384e641208f260","failure_pattern":"ERROR CODE: 429; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: 429","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.814663951120163,"anchor_occurrences":4,"after_last_push_count":5,"trend_counts":[0,0,1,1,1,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-09-28"],"prior_runs_affected":9,"prior_last_seen_at":"2026-10-01T23:36:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:108]: failed to get service managed identity\nUnexpected error:\n    \u003c*exported.ResponseError | 0x31fef30d4a50\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-04-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-04-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n            correlation ID: 63080195-ea99-42b1-8144-1ce2de77a2d8\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:108]: failed to get service managed identity\nUnexpected error:\n    \u003c*exported.ResponseError | 0x31fef30d4a50\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-04-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-04-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n            correlation ID: 63080195-ea99-42b1-8144-1ce2de77a2d8\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:108]: failed to get service managed identity\nUnexpected error:\n    \u003c*exported.ResponseError | 0xf085da10420\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because tenant XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n            correlation ID: 34115d09-ce39-4c55-8529-6ab72e73b469\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:108]: failed to get service managed identity\nUnexpected error:\n    \u003c*exported.ResponseError | 0xf085da10420\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because tenant XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n            correlation ID: 34115d09-ce39-4c55-8529-6ab72e73b469\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:119]: failed to get cluster-api-azure managed identity\nUnexpected error:\n    \u003c*exported.ResponseError | 0x18d22f431050\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n            correlation ID: d277ac90-384a-44a9-9799-7698621627f9\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:119]: failed to get cluster-api-azure managed identity\nUnexpected error:\n    \u003c*exported.ResponseError | 0x18d22f431050\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n            correlation ID: d277ac90-384a-44a9-9799-7698621627f9\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7523/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111072522505031680","occurred_at":"2026-10-16T12:31:25Z","signature_id":"78562743ea1326baf0fb7954282c6ad239a40798722c5ed5c25f650423266f08","pr_number":7523,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558411769778176","occurred_at":"2026-10-15T02:28:32Z","signature_id":"48846a4a9ee7ad69b6a19d79cdae37682330e1e94cef9f146c300baeb116f561","pr_number":7342,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110168597706838016","occurred_at":"2026-10-14T00:39:33Z","signature_id":"e91976aae7db2919970076bbc44f0d95243241475acf02a79c2cb09c59930f22","pr_number":7488,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110099886413189120","occurred_at":"2026-10-13T20:04:06Z","signature_id":"e835da002c88402841b37e84e6cf6143f8cac4fd26e56433266edb67a38bff39","pr_number":7480,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a36cdec30df49971b11d0532fa3d90b6d04db004910c6c9ba6b1036a99dab7a3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.814663951120163,"anchor_occurrences":4,"after_last_push_count":3,"trend_counts":[0,2,0,1,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30ba0d996320\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-kv7nqskz8rbt\" to finish creating: GET https://rp.j0712192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf68877b-df8e-430d-ab1f-aa5c781ac5c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf68877b-df8e-430d-ab1f-aa5c781ac5c2\",\n      \"name\": \"bf68877b-df8e-430d-ab1f-aa5c781ac5c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T14:23:20.152433644Z\",\n      \"endTime\": \"2026-10-15T14:52:22.758892045Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-7zsxm-lwj52: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-kv7nqskz8rbt--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-7zsxm-lwj52 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-7zsxm-lwj52: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-kv7nqskz8rbt--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-7zsxm-lwj52 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0712192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf68877b-df8e-430d-ab1f-aa5c781ac5c2\n            correlation ID: 6f1d82bf-560b-4ab8-8830-31f9f4f34d95\n            client request ID: f649116d-c373-4f92-866d-87f54a1d5dfb\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30ba0d996320\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-kv7nqskz8rbt\" to finish creating: GET https://rp.j0712192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf68877b-df8e-430d-ab1f-aa5c781ac5c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf68877b-df8e-430d-ab1f-aa5c781ac5c2\",\n      \"name\": \"bf68877b-df8e-430d-ab1f-aa5c781ac5c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T14:23:20.152433644Z\",\n      \"endTime\": \"2026-10-15T14:52:22.758892045Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-7zsxm-lwj52: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-kv7nqskz8rbt--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-7zsxm-lwj52 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-7zsxm-lwj52: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-kv7nqskz8rbt--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-7zsxm-lwj52 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0712192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf68877b-df8e-430d-ab1f-aa5c781ac5c2\n            correlation ID: 6f1d82bf-560b-4ab8-8830-31f9f4f34d95\n            client request ID: f649116d-c373-4f92-866d-87f54a1d5dfb\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22a4be5cfc40\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-gc2cf577p6fs\" to finish creating: GET https://rp.j0375552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1dec4678-5a13-4201-9600-24dcb5a26534\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1dec4678-5a13-4201-9600-24dcb5a26534\",\n      \"name\": \"1dec4678-5a13-4201-9600-24dcb5a26534\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T16:17:52.259133125Z\",\n      \"endTime\": \"2026-10-14T16:46:52.310381675Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-dnjw8-hs9xx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-gc2cf577p6fs--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-dnjw8-hs9xx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-dnjw8-hs9xx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-gc2cf577p6fs--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-dnjw8-hs9xx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0375552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1dec4678-5a13-4201-9600-24dcb5a26534\n            correlation ID: b81493f7-b048-4704-b54a-5a69ef8ad3b5\n            client request ID: 4dd74744-6a7e-48d1-9b1b-985d5238151a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22a4be5cfc40\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-gc2cf577p6fs\" to finish creating: GET https://rp.j0375552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1dec4678-5a13-4201-9600-24dcb5a26534\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1dec4678-5a13-4201-9600-24dcb5a26534\",\n      \"name\": \"1dec4678-5a13-4201-9600-24dcb5a26534\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T16:17:52.259133125Z\",\n      \"endTime\": \"2026-10-14T16:46:52.310381675Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-dnjw8-hs9xx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-gc2cf577p6fs--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-dnjw8-hs9xx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-dnjw8-hs9xx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-gc2cf577p6fs--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-dnjw8-hs9xx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0375552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1dec4678-5a13-4201-9600-24dcb5a26534\n            correlation ID: b81493f7-b048-4704-b54a-5a69ef8ad3b5\n            client request ID: 4dd74744-6a7e-48d1-9b1b-985d5238151a\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x194566f239e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-smsvt85kmkqr\" to finish creating: GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\",\n      \"name\": \"0da32617-05fc-4b99-8fa9-4a08b5bd5119\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T19:35:51.873467614Z\",\n      \"endTime\": \"2026-10-12T20:04:52.500489339Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-4gnlk-rqpw4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-smsvt85kmkqr--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-4gnlk-rqpw4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-4gnlk-rqpw4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-smsvt85kmkqr--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-4gnlk-rqpw4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\n            correlation ID: 6e6e9b92-3b22-4eef-a78b-96ae21c44dbf\n            client request ID: d66a3161-f2e3-4d33-bb73-476a2a9b1435\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x194566f239e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-smsvt85kmkqr\" to finish creating: GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\",\n      \"name\": \"0da32617-05fc-4b99-8fa9-4a08b5bd5119\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T19:35:51.873467614Z\",\n      \"endTime\": \"2026-10-12T20:04:52.500489339Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-4gnlk-rqpw4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-smsvt85kmkqr--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-4gnlk-rqpw4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-4gnlk-rqpw4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-smsvt85kmkqr--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-4gnlk-rqpw4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\n            correlation ID: 6e6e9b92-3b22-4eef-a78b-96ae21c44dbf\n            client request ID: d66a3161-f2e3-4d33-bb73-476a2a9b1435\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110717569160712192","occurred_at":"2026-10-15T13:00:26Z","signature_id":"c8417503845a3355ef92434db4574a991df797fd4b28c39199027f6764816bf9","pr_number":7480,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110389077940375552","occurred_at":"2026-10-14T15:15:39Z","signature_id":"a910098321a76c4517d5d09d23a0ef9fa638d78b5f1bc6f25a192f2153e44e26","pr_number":7492,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109715328622661632","occurred_at":"2026-10-12T18:38:25Z","signature_id":"0b8a3b5437beafe0105aa62d86153fdc67ea6aaf34efdb9197853845e0812245","pr_number":7479,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7420/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109588388414754816","occurred_at":"2026-10-12T10:14:00Z","signature_id":"810897085d9afd12e972e9199e65575a09acca1a53e133ff2eb94e81c5444daf","pr_number":7420,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1a53b0c30b515bff265c32e9b65ff52445c52775e0172c5df823b4597bd77e2b","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.814663951120163,"anchor_occurrences":4,"after_last_push_count":1,"trend_counts":[0,0,1,2,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-09T11:02:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x1ff649812120\u003e: \n    resource group \"rg-candidate-4-20-ffzv544l6w92\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-pf2tq2\" in resourcegroup=\"rg-candidate-4-20-ffzv544l6w92\" to finish deleting: GET https://rp.j4344192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/08aca983-942f-491c-b751-ff15b4ba5c58\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/08aca983-942f-491c-b751-ff15b4ba5c58\",\n      \"name\": \"08aca983-942f-491c-b751-ff15b4ba5c58\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T11:21:34.419311778Z\",\n      \"endTime\": \"2026-10-15T11:45:37.398768071Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4344192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/08aca983-942f-491c-b751-ff15b4ba5c58\n            correlation ID: 34b324bb-a098-46c8-8e16-84d85878762f\n            client request ID: 039d8e60-0e2e-4da7-ae0f-2e1857606679\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x1ff649812120\u003e: \n    resource group \"rg-candidate-4-20-ffzv544l6w92\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-pf2tq2\" in resourcegroup=\"rg-candidate-4-20-ffzv544l6w92\" to finish deleting: GET https://rp.j4344192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/08aca983-942f-491c-b751-ff15b4ba5c58\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/08aca983-942f-491c-b751-ff15b4ba5c58\",\n      \"name\": \"08aca983-942f-491c-b751-ff15b4ba5c58\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T11:21:34.419311778Z\",\n      \"endTime\": \"2026-10-15T11:45:37.398768071Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4344192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/08aca983-942f-491c-b751-ff15b4ba5c58\n            correlation ID: 34b324bb-a098-46c8-8e16-84d85878762f\n            client request ID: 039d8e60-0e2e-4da7-ae0f-2e1857606679\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:320]: Unexpected error:\n    \u003c*errors.joinError | 0x1e1ed182150\u003e: \n    resource group \"rg-candidate-4-21-xnx2svpmndkd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-bqkz5m\" in resourcegroup=\"rg-candidate-4-21-xnx2svpmndkd\" to finish deleting: GET https://rp.j6397952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aefc9a01-d084-42f4-9d9d-673831774ba7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aefc9a01-d084-42f4-9d9d-673831774ba7\",\n      \"name\": \"aefc9a01-d084-42f4-9d9d-673831774ba7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T20:07:21.117969363Z\",\n      \"endTime\": \"2026-10-14T20:31:23.904548294Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6397952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aefc9a01-d084-42f4-9d9d-673831774ba7\n            correlation ID: e7f218c6-4d7a-4ce1-8044-c7ec05ec7356\n            client request ID: e554513d-44da-4c58-8dae-c72863b0374d\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:320]: Unexpected error:\n    \u003c*errors.joinError | 0x1e1ed182150\u003e: \n    resource group \"rg-candidate-4-21-xnx2svpmndkd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-bqkz5m\" in resourcegroup=\"rg-candidate-4-21-xnx2svpmndkd\" to finish deleting: GET https://rp.j6397952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aefc9a01-d084-42f4-9d9d-673831774ba7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aefc9a01-d084-42f4-9d9d-673831774ba7\",\n      \"name\": \"aefc9a01-d084-42f4-9d9d-673831774ba7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T20:07:21.117969363Z\",\n      \"endTime\": \"2026-10-14T20:31:23.904548294Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6397952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aefc9a01-d084-42f4-9d9d-673831774ba7\n            correlation ID: e7f218c6-4d7a-4ce1-8044-c7ec05ec7356\n            client request ID: e554513d-44da-4c58-8dae-c72863b0374d\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x50b30376378\u003e: \n    resource group \"rg-candidate-4-23-zgqgjh9b82nm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-v7mn4s\" in resourcegroup=\"rg-candidate-4-23-zgqgjh9b82nm\" to finish deleting: GET https://rp.j9234816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90b34b3b-49e0-48b7-b175-fed3e7efcffe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90b34b3b-49e0-48b7-b175-fed3e7efcffe\",\n      \"name\": \"90b34b3b-49e0-48b7-b175-fed3e7efcffe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T15:55:23.629401759Z\",\n      \"endTime\": \"2026-10-14T16:19:25.512560555Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9234816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90b34b3b-49e0-48b7-b175-fed3e7efcffe\n            correlation ID: 18b3d50f-1bf0-4dc5-95df-d87efa3be15d\n            client request ID: eef778ec-0385-499a-a74c-a4ea9c9eb634\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x50b30376378\u003e: \n    resource group \"rg-candidate-4-23-zgqgjh9b82nm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-v7mn4s\" in resourcegroup=\"rg-candidate-4-23-zgqgjh9b82nm\" to finish deleting: GET https://rp.j9234816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90b34b3b-49e0-48b7-b175-fed3e7efcffe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90b34b3b-49e0-48b7-b175-fed3e7efcffe\",\n      \"name\": \"90b34b3b-49e0-48b7-b175-fed3e7efcffe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T15:55:23.629401759Z\",\n      \"endTime\": \"2026-10-14T16:19:25.512560555Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9234816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90b34b3b-49e0-48b7-b175-fed3e7efcffe\n            correlation ID: 18b3d50f-1bf0-4dc5-95df-d87efa3be15d\n            client request ID: eef778ec-0385-499a-a74c-a4ea9c9eb634\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7522/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110675476174344192","occurred_at":"2026-10-15T10:13:41Z","signature_id":"16f7a4a75b4f215c79e6e70dbd9f1ff1bdef47107a0db62e0c2e09b4544ba706","pr_number":7522,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110445968326397952","occurred_at":"2026-10-14T19:01:26Z","signature_id":"9f4ffc5eb425744bf1f26b0e1f04b2d4faa6a9bb922b93d34f3023ef78ad7eed","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7428/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110376503379234816","occurred_at":"2026-10-14T14:25:41Z","signature_id":"f07da3d933a833699409e75c9aa7df75da760b199cbfea3580c7225c9d2f11ef","pr_number":7428,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110098887174787072","occurred_at":"2026-10-13T19:57:04Z","signature_id":"181ae70b8645dc86d286ecef9ed77dae1b64d0e1fdc182c681d11979a0362e0d","pr_number":7503,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.814663951120163,"anchor_occurrences":4,"after_last_push_count":7,"trend_counts":[1,1,1,1,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":17,"prior_last_seen_at":"2026-10-11T23:05:50Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1cdd9060dca0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-lqb44pqrvnjn\" to finish creating: GET https://rp.j3413376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1eb1b48b-885e-44b5-a938-88689e5212e6\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/1eb1b48b-885e-44b5-a938-88689e5212e6' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1eb1b48b-885e-44b5-a938-88689e5212e6\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3413376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1eb1b48b-885e-44b5-a938-88689e5212e6\n            correlation ID: 70fa4741-a1d2-4d88-b00c-c2caa1e45683\n            client request ID: c17bea04-7230-45d2-bf8b-2cdbaefe70c1\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1cdd9060dca0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-lqb44pqrvnjn\" to finish creating: GET https://rp.j3413376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1eb1b48b-885e-44b5-a938-88689e5212e6\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/1eb1b48b-885e-44b5-a938-88689e5212e6' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1eb1b48b-885e-44b5-a938-88689e5212e6\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3413376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1eb1b48b-885e-44b5-a938-88689e5212e6\n            correlation ID: 70fa4741-a1d2-4d88-b00c-c2caa1e45683\n            client request ID: c17bea04-7230-45d2-bf8b-2cdbaefe70c1\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x12acc23ba300\u003e: \n    failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-kwgn8mlj4cqw\" to finish creating: GET https://rp.j2922112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2922112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64\n            correlation ID: 6cf5166e-d7f7-4d42-8f52-96bfabf5f7df\n            client request ID: d4b2a1df-961f-48a7-9e13-886e82b9c210\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x12acc23ba300\u003e: \n    failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-kwgn8mlj4cqw\" to finish creating: GET https://rp.j2922112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2922112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64\n            correlation ID: 6cf5166e-d7f7-4d42-8f52-96bfabf5f7df\n            client request ID: d4b2a1df-961f-48a7-9e13-886e82b9c210\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x343c841917c0\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-ld8wft\" in resourcegroup=\"rg-np-noedge-ld8wft-lrx6cgcqbn7h\" to finish creating: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6\n            correlation ID: ffecaa7d-bd14-4598-8eba-697dbec64dbe\n            client request ID: 41765bff-079d-45be-ae54-98a967650838\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x343c841917c0\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-ld8wft\" in resourcegroup=\"rg-np-noedge-ld8wft-lrx6cgcqbn7h\" to finish creating: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6\n            correlation ID: ffecaa7d-bd14-4598-8eba-697dbec64dbe\n            client request ID: 41765bff-079d-45be-ae54-98a967650838\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7532/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110649423133413376","occurred_at":"2026-10-15T08:30:11Z","signature_id":"1ab688bb4d985b93965cfa3884a09f516dd87f67ad6ea4b16b6829b9af396c3d","pr_number":7532,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110437162552922112","occurred_at":"2026-10-14T18:26:44Z","signature_id":"d7964fba6ca87a857bce8a35d83b03910a03ed75438c875148bf5ec27b0319d1","pr_number":7524,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"8d5ca072ced2a86e224e1159c22de688aa029c8ce11fd664e12553f4ed9b8ecd","pr_number":7496,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109536551896616960","occurred_at":"2026-10-12T06:48:01Z","signature_id":"9b6a19c4bbd8e0fee198397011d61bfa3837fe3c4ea1afea5bc1a3c2f92f5308","pr_number":7440,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9f75b69b94f5e69dd2ea89d590d15c51614ce3eda1965d4efc965f1caf5f8e2b","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption Burst High - \u003ccosmos-account\u003e fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption Burst High - \u003ccosmos-account\u003e fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption Burst High - arohcpci01-rp-j0532352 does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.814663951120163,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,3,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption Burst High - arohcpci01-rp-j0532352 does not fire","occurrences":1},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption Burst High - arohcpci01-rp-j2816128 does not fire","occurrences":1},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption Burst High - arohcpci01-rp-j6775168 does not fire","occurrences":1},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption Burst High - arohcpci01-rp-j7369856 does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-14T03:40:18Z\n  Ended:       2026-10-14T03:43:19Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 95% averaged over a 5-minute window, evaluated every minute. This catches short RU-saturation bursts (e.g. mass cluster creation at the start of E2E tests) that exceed provisioned throughput and can lead to 429 throttling before the 10-minute sustained alert fires. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-14T03:52:13Z\n  Ended:       2026-10-14T04:06:24Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 95% averaged over a 5-minute window, evaluated every minute. This catches short RU-saturation bursts (e.g. mass cluster creation at the start of E2E tests) that exceed provisioned throughput and can lead to 429 throttling before the 10-minute sustained alert fires. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-14T00:40:15Z\n  Ended:       2026-10-14T01:10:12Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 95% averaged over a 5-minute window, evaluated every minute. This catches short RU-saturation bursts (e.g. mass cluster creation at the start of E2E tests) that exceed provisioned throughput and can lead to 429 throttling before the 10-minute sustained alert fires. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-13T22:03:21Z\n  Ended:       2026-10-13T22:29:18Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 95% averaged over a 5-minute window, evaluated every minute. This catches short RU-saturation bursts (e.g. mass cluster creation at the start of E2E tests) that exceed provisioned throughput and can lead to 429 throttling before the 10-minute sustained alert fires. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110199338117369856","occurred_at":"2026-10-14T02:41:42Z","signature_id":"517b35de6d4d5ec7325035eafd6825972465809bddcc7128c935e22b8d7cb4df","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110153446450532352","occurred_at":"2026-10-13T23:39:20Z","signature_id":"3845ce22d709f7d6f88c6b78d29a932df4b7e3435bd9d98ded715b56bb04b04e","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110115702512816128","occurred_at":"2026-10-13T21:09:22Z","signature_id":"3845ce22d709f7d6f88c6b78d29a932df4b7e3435bd9d98ded715b56bb04b04e","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110080447856775168","occurred_at":"2026-10-13T17:40:39Z","signature_id":"3845ce22d709f7d6f88c6b78d29a932df4b7e3435bd9d98ded715b56bb04b04e","pr_number":7495,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=1)","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.814663951120163,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,0,0,2,2,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=1)","occurrences":3},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-16T23:22:36.468Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-10-16T23:22:36.986Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:d3bdf5672c862ebc0a35ea497266f8c7598862b9377e076ed14ffc329be19981]\"\ntime=2026-10-16T23:22:36.986Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-16T23:22:37.251Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 namespace=default-sc\ntime=2026-10-16T23:22:37.674Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 dryRun=false\ntime=2026-10-16T23:22:37.674Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-16T23:22:37.803Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-16T23:23:44.090Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-16T23:23:44.090Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"","time=2026-10-16T15:46:44.545Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-10-16T15:46:45.068Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:d3bdf5672c862ebc0a35ea497266f8c7598862b9377e076ed14ffc329be19981]\"\ntime=2026-10-16T15:46:45.068Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-16T15:46:45.193Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 namespace=default-sc\ntime=2026-10-16T15:46:45.343Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 dryRun=false\ntime=2026-10-16T15:46:45.343Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-16T15:46:45.452Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-16T15:47:32.298Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-16T15:47:32.298Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"","time=2026-10-15T23:49:29.451Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-10-15T23:49:29.912Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:d3bdf5672c862ebc0a35ea497266f8c7598862b9377e076ed14ffc329be19981]\"\ntime=2026-10-15T23:49:29.912Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-15T23:49:30.191Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 namespace=default-sc\ntime=2026-10-15T23:49:30.484Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 dryRun=false\ntime=2026-10-15T23:49:30.485Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-15T23:49:30.579Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-15T23:50:28.126Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-15T23:50:28.126Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111228423002132480","occurred_at":"2026-10-16T22:50:55Z","signature_id":"bda1134e26c870bac63ca65499e242da0400c142162bed259a46288d26e4a236","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111110619204685824","occurred_at":"2026-10-16T15:02:46Z","signature_id":"19498df56e93c37fe7f6fb08ad99bc2b1b1c8d05100b0a59886217e4fe1408e0","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110868757877362688","occurred_at":"2026-10-15T23:01:44Z","signature_id":"bda1134e26c870bac63ca65499e242da0400c142162bed259a46288d26e4a236","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110747548531560448","occurred_at":"2026-10-15T15:00:05Z","signature_id":"bda1134e26c870bac63ca65499e242da0400c142162bed259a46288d26e4a236","pr_number":7451,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fa106de26c29607201ec4aa06c503cd9db99f65136c9944baef8962062de09a9","failure_pattern":"VerifyHyperShiftIngressCertificate timed out after 10m0","search_query":"VerifyHyperShiftIngressCertificate timed out after 10m0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,4,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-08T07:02:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1b7c5f7105e0\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x1b7c6068e900\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1b7c5f7105e0\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x1b7c6068e900\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x14a7efc6800\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x14a7fffde80\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x14a7efc6800\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x14a7fffde80\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x194b8fb466e0\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x194b8fb466c0\u003e{\n            msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x194b8fb466a0\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x194b8fb466e0\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x194b8fb466c0\u003e{\n            msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x194b8fb466a0\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","signature_id":"5c557e1356eb3b63a8003453927c70609ed1a2e90406457fdd54fc88e1fb53e5","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","signature_id":"6334833f41bf18b21c6f6a2861528f92d6fb485ca2f0d52c7d87545a02b6cbc9","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","signature_id":"c15f77333166c293b45237bf13679bc471c7c7da27a687c44335f6bfc722d028","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","signature_id":"f778b2e59ac991a933120117b78a9c24dc834541831bf4f98f55f52a6dba1078","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0ab80c2f2404154d860a57027c4c22b196a560eb7d17aca0414b079cdbdcf0e8","failure_pattern":"VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s","search_query":"VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.22","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,0,4,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x26b7f5580860\u003e: \n    VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\n    {\n        s: \"VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x26b7f5580860\u003e: \n    VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\n    {\n        s: \"VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x5b6d94696d0\u003e: \n    VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\n    {\n        s: \"VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x5b6d94696d0\u003e: \n    VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\n    {\n        s: \"VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x362088f5cd60\u003e: \n    VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\n    {\n        s: \"VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x362088f5cd60\u003e: \n    VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\n    {\n        s: \"VerifyIngressControllerScope(External): timeout must be \u003e 0, got 0s\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"12a803c93bfc31c149c19e19babb26d0b3c6504666182198eaa16e7612f760e8","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"37bcf36d54f44bfeee035bd8567e75aee5c0283f942d5d8f119f3f70bcf86f25","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"43c0a9246f132ce547efe4ef404ff816fd70dec0983aadabc64c9b64714c506a","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340691224694784","occurred_at":"2026-10-14T12:02:45Z","signature_id":"5a3d8ec5914654258d035b53c13a9219b5025b4a9e88fc022e31972f06ab9a6c","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.6109979633401222,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[0,1,1,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":6,"prior_last_seen_at":"2026-09-30T16:03:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0x175f66e5a600\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-8brvsr9whg7b\" to finish creating: GET https://rp.j2105600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cba8c76e-06a0-4c2a-b3e0-d699a7abb497\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cba8c76e-06a0-4c2a-b3e0-d699a7abb497\",\n      \"name\": \"cba8c76e-06a0-4c2a-b3e0-d699a7abb497\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T13:47:00.778542361Z\",\n      \"endTime\": \"2026-10-14T14:16:06.716990034Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.37 is Partial (want Completed), started 17m32s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2105600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cba8c76e-06a0-4c2a-b3e0-d699a7abb497\n            correlation ID: 113b689f-35da-4e51-8207-8a80d51d19ea\n            client request ID: d9df4b89-ae88-4991-9a25-651fec75b816\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0x175f66e5a600\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-8brvsr9whg7b\" to finish creating: GET https://rp.j2105600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cba8c76e-06a0-4c2a-b3e0-d699a7abb497\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cba8c76e-06a0-4c2a-b3e0-d699a7abb497\",\n      \"name\": \"cba8c76e-06a0-4c2a-b3e0-d699a7abb497\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T13:47:00.778542361Z\",\n      \"endTime\": \"2026-10-14T14:16:06.716990034Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.37 is Partial (want Completed), started 17m32s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2105600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cba8c76e-06a0-4c2a-b3e0-d699a7abb497\n            correlation ID: 113b689f-35da-4e51-8207-8a80d51d19ea\n            client request ID: d9df4b89-ae88-4991-9a25-651fec75b816\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2219287fa880\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-v5gf99nj48r6\" to finish creating: GET https://rp.j0001920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2199d8bb-869a-41e2-acec-31983e058033\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2199d8bb-869a-41e2-acec-31983e058033\",\n      \"name\": \"2199d8bb-869a-41e2-acec-31983e058033\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T11:37:06.655702009Z\",\n      \"endTime\": \"2026-10-13T12:06:15.607847324Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.18 is Partial (want Completed), started 8m45s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0001920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2199d8bb-869a-41e2-acec-31983e058033\n            correlation ID: 436fe8be-4edd-4c43-bff6-c5ba6dee20f5\n            client request ID: 88121473-7bcb-4b6c-858a-daddb3fc95db\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2219287fa880\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-v5gf99nj48r6\" to finish creating: GET https://rp.j0001920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2199d8bb-869a-41e2-acec-31983e058033\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2199d8bb-869a-41e2-acec-31983e058033\",\n      \"name\": \"2199d8bb-869a-41e2-acec-31983e058033\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T11:37:06.655702009Z\",\n      \"endTime\": \"2026-10-13T12:06:15.607847324Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.18 is Partial (want Completed), started 8m45s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0001920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2199d8bb-869a-41e2-acec-31983e058033\n            correlation ID: 436fe8be-4edd-4c43-bff6-c5ba6dee20f5\n            client request ID: 88121473-7bcb-4b6c-858a-daddb3fc95db\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0x36d0cb4c81e0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-z7qkjc46tf97\" to finish creating: GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\",\n      \"name\": \"d6731312-0cc9-4e1d-a296-bfc0d751168f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T18:45:24.479848853Z\",\n      \"endTime\": \"2026-10-12T19:14:30.222274838Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 19m29s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\n            correlation ID: 4572b63a-1060-450d-b5db-9eae5d0fdb78\n            client request ID: 5f229b06-fd04-479f-bdad-788fe5458949\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0x36d0cb4c81e0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-z7qkjc46tf97\" to finish creating: GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\",\n      \"name\": \"d6731312-0cc9-4e1d-a296-bfc0d751168f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T18:45:24.479848853Z\",\n      \"endTime\": \"2026-10-12T19:14:30.222274838Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 19m29s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\n            correlation ID: 4572b63a-1060-450d-b5db-9eae5d0fdb78\n            client request ID: 5f229b06-fd04-479f-bdad-788fe5458949\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7522/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110350863582105600","occurred_at":"2026-10-14T12:43:47Z","signature_id":"fdd70b6d05559ad6a1e3f80813310696d29fbbfa70255c4f081213256f2c33ce","pr_number":7522,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109958187410001920","occurred_at":"2026-10-13T10:43:27Z","signature_id":"7322f74e045c9377f4c356e763fe2af1a5554b1f96d50357c44c7e7728ee62bb","pr_number":7488,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109704412896169984","occurred_at":"2026-10-12T17:55:03Z","signature_id":"317dedee167ea472d7b8eb2f6ca44fba55d08af18e00d84e205137d4efe864b6","pr_number":7467,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a1a1afad2dc1f0ee985d82cc0c7931dff52aae3d7be3427571c055d66528c857","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.6109979633401222,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[0,3,0,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-10T11:45:56Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x6f1edda03a0\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-jdq4zztv647q\" to finish creating: GET https://rp.j2613760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\",\n      \"name\": \"97c24699-e347-4966-b180-bf6ae5ff9b03\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T21:54:28.61846625Z\",\n      \"endTime\": \"2026-10-12T21:54:42.231408957Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-jdq4zztv647q/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2613760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\n            correlation ID: 8f0a4fda-224a-436d-84e3-d641535231f1\n            client request ID: d44e41d0-633f-4992-a25b-ce6473619f8a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x6f1edda03a0\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-jdq4zztv647q\" to finish creating: GET https://rp.j2613760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\",\n      \"name\": \"97c24699-e347-4966-b180-bf6ae5ff9b03\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T21:54:28.61846625Z\",\n      \"endTime\": \"2026-10-12T21:54:42.231408957Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-jdq4zztv647q/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2613760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\n            correlation ID: 8f0a4fda-224a-436d-84e3-d641535231f1\n            client request ID: d44e41d0-633f-4992-a25b-ce6473619f8a\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x10139db40a0\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-v52tck7hh95p\" to finish creating: GET https://rp.j8314496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\",\n      \"name\": \"567cdaaa-ef82-48d8-9481-2e386ea53f4d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T21:36:04.986032041Z\",\n      \"endTime\": \"2026-10-12T21:36:44.722884546Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-v52tck7hh95p/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'ccbf438f-08ec-4e2b-ae4e-219ce7dc3762' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8314496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\n            correlation ID: 7bddac41-f6a3-4695-b95b-13972ea0840e\n            client request ID: ab9e631f-f159-4b76-a0df-2b299f43d7d8\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x10139db40a0\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-v52tck7hh95p\" to finish creating: GET https://rp.j8314496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\",\n      \"name\": \"567cdaaa-ef82-48d8-9481-2e386ea53f4d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T21:36:04.986032041Z\",\n      \"endTime\": \"2026-10-12T21:36:44.722884546Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-v52tck7hh95p/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'ccbf438f-08ec-4e2b-ae4e-219ce7dc3762' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8314496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\n            correlation ID: 7bddac41-f6a3-4695-b95b-13972ea0840e\n            client request ID: ab9e631f-f159-4b76-a0df-2b299f43d7d8\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32753cabb000\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-lcmmtbwtg46t\" to finish creating: GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\",\n      \"name\": \"b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T09:39:05.150325854Z\",\n      \"endTime\": \"2026-10-12T09:39:46.941270418Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\n            correlation ID: 84029b8a-bf49-4092-9e8c-07ae2562e0b2\n            client request ID: 168cc3ec-3b47-4824-95c6-847f6de09edb\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32753cabb000\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-lcmmtbwtg46t\" to finish creating: GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\",\n      \"name\": \"b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T09:39:05.150325854Z\",\n      \"endTime\": \"2026-10-12T09:39:46.941270418Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\n            correlation ID: 84029b8a-bf49-4092-9e8c-07ae2562e0b2\n            client request ID: 168cc3ec-3b47-4824-95c6-847f6de09edb\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7428/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109749870842613760","occurred_at":"2026-10-12T20:55:41Z","signature_id":"69699f183ac245894ec15c2257ce86fb84bc1f5bed2cf85b38d4c739dedbd9d1","pr_number":7428,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109742847518314496","occurred_at":"2026-10-12T20:27:46Z","signature_id":"d4b16f343401a26bebbb19567d2aeea68fce56219f43f4bb17023f4763f88712","pr_number":7397,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","signature_id":"f06a1fb8a7fb87970f3aa02a8622a506c32fd712c86f231325a5e5fce4359407","pr_number":7463,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Insights","search_query":"ERROR CODE: ResourceGroupNotFound","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert collection is complete","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.6109979633401222,"anchor_occurrences":3,"after_last_push_count":4,"trend_counts":[0,2,1,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-09T09:35:37Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert collection is complete","occurrences":3}],"full_error_samples":["alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"ResourceGroupNotFound\",\n    \"message\": \"Resource group 'hcp-underlay-ci01-usw3' could not be found.\"\n  }\n}\n--------------------------------------------------------------------------------","alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9114496: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9114496/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"ResourceGroupNotFound\",\n    \"message\": \"Resource group 'hcp-underlay-ci01-j9114496' could not be found.\"\n  }\n}\n--------------------------------------------------------------------------------","alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2837760: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2837760/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"ResourceGroupNotFound\",\n    \"message\": \"Resource group 'hcp-underlay-ci01-j2837760' could not be found.\"\n  }\n}\n--------------------------------------------------------------------------------"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7501/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110030920055525376","occurred_at":"2026-10-13T15:32:24Z","signature_id":"a13b64d94d488082a4a9ffd8fdeb35b5d5d0a71c93381c3fedbe0bba79b488d1","pr_number":7501,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109792838379114496","occurred_at":"2026-10-12T23:46:25Z","signature_id":"7dde37c662e4768f610ecb85052e99f87f88976e1e543acbe408c61888caa61d","pr_number":7439,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109727639982837760","occurred_at":"2026-10-12T19:27:20Z","signature_id":"3223dd11b35ca881a6905f2213555998ec56974987e667be15a6afbf19cd4f19","pr_number":6920,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.6109979633401222,"anchor_occurrences":3,"after_last_push_count":7,"trend_counts":[1,2,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":28,"prior_last_seen_at":"2026-10-11T19:48:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:467]: Timed out after 600.003s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x23efaa396df0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:467]: Timed out after 600.003s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x23efaa396df0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:412]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x8ceb7456d30\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:412]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x8ceb7456d30\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.007s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x23913a80e730\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.007s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x23913a80e730\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111090334573268992","occurred_at":"2026-10-16T13:42:12Z","signature_id":"013f773a3c97f51390c5446d7bcbcb1250a51911bae762a125e99650623c4fcd","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"f29a7407248c09d9f2d63fc02a535acb501328e35c4bfacc81f04e0e98c0e854","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109660658009116672","occurred_at":"2026-10-12T15:00:29Z","signature_id":"f0b4f980a6b881fe20a87e1354e5792bdbf23c290ec125adfa5d9d993ae815ea","pr_number":7467,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":3,"after_last_push_count":7,"trend_counts":[0,0,3,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":19,"prior_last_seen_at":"2026-10-09T11:00:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x223786a70108\u003e: \n    resource group \"rg-candidate-5-0-l4hscszf2rxz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-lvqr8d\" in resourcegroup=\"rg-candidate-5-0-l4hscszf2rxz\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b44ea302-7f97-4969-a60c-eec855852cac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b44ea302-7f97-4969-a60c-eec855852cac\",\n      \"name\": \"b44ea302-7f97-4969-a60c-eec855852cac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T16:40:42.747363753Z\",\n      \"endTime\": \"2026-10-13T17:04:51.623105937Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o054eslakn49l68pthqigeds4ule still exists (deletion dispatched at 2026-10-13T16:40:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b44ea302-7f97-4969-a60c-eec855852cac\n            correlation ID: 4bfc0959-e629-40fb-9245-01ec4651bd6b\n            client request ID: a781f5bf-303e-4280-ba8a-44cc833b6ce4\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x223786a70108\u003e: \n    resource group \"rg-candidate-5-0-l4hscszf2rxz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-lvqr8d\" in resourcegroup=\"rg-candidate-5-0-l4hscszf2rxz\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b44ea302-7f97-4969-a60c-eec855852cac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b44ea302-7f97-4969-a60c-eec855852cac\",\n      \"name\": \"b44ea302-7f97-4969-a60c-eec855852cac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T16:40:42.747363753Z\",\n      \"endTime\": \"2026-10-13T17:04:51.623105937Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o054eslakn49l68pthqigeds4ule still exists (deletion dispatched at 2026-10-13T16:40:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b44ea302-7f97-4969-a60c-eec855852cac\n            correlation ID: 4bfc0959-e629-40fb-9245-01ec4651bd6b\n            client request ID: a781f5bf-303e-4280-ba8a-44cc833b6ce4\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x34e38763a138\u003e: \n    resource group \"rg-cordon-test-9l4cgfh8mhfl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-9l4cgfh8mhfl\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ab95bc-886b-4b3d-9a6e-9221165e46bd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ab95bc-886b-4b3d-9a6e-9221165e46bd\",\n      \"name\": \"84ab95bc-886b-4b3d-9a6e-9221165e46bd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T16:39:24.254400328Z\",\n      \"endTime\": \"2026-10-13T17:03:31.595626382Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o1agno7465r2vsakhkack392k7ql still exists (deletion dispatched at 2026-10-13T16:39:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ab95bc-886b-4b3d-9a6e-9221165e46bd\n            correlation ID: 9b32bec0-6d45-4533-a8ad-d586fd5de17c\n            client request ID: 0c2d0a32-aa6c-4c82-a1b0-303746bfb85e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x34e38763a138\u003e: \n    resource group \"rg-cordon-test-9l4cgfh8mhfl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-9l4cgfh8mhfl\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ab95bc-886b-4b3d-9a6e-9221165e46bd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ab95bc-886b-4b3d-9a6e-9221165e46bd\",\n      \"name\": \"84ab95bc-886b-4b3d-9a6e-9221165e46bd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T16:39:24.254400328Z\",\n      \"endTime\": \"2026-10-13T17:03:31.595626382Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o1agno7465r2vsakhkack392k7ql still exists (deletion dispatched at 2026-10-13T16:39:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ab95bc-886b-4b3d-9a6e-9221165e46bd\n            correlation ID: 9b32bec0-6d45-4533-a8ad-d586fd5de17c\n            client request ID: 0c2d0a32-aa6c-4c82-a1b0-303746bfb85e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x39bb325a6558\u003e: \n    resource group \"admin-api-serialconsole-dwh828chkx9c\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-dwh828chkx9c\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0074eb0c-2b24-41cb-a7ff-604c4e76ad09\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0074eb0c-2b24-41cb-a7ff-604c4e76ad09\",\n      \"name\": \"0074eb0c-2b24-41cb-a7ff-604c4e76ad09\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T16:33:20.010489415Z\",\n      \"endTime\": \"2026-10-13T16:57:21.684870881Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0nldrfocqarmt0r34ht6jph1m9d19 still exists (deletion dispatched at 2026-10-13T16:33:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0074eb0c-2b24-41cb-a7ff-604c4e76ad09\n            correlation ID: d6c08569-5a60-4a0f-bb76-207075ebe9e3\n            client request ID: f4713fe8-8c37-423d-a5df-6a24a2af5b2d\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x39bb325a6558\u003e: \n    resource group \"admin-api-serialconsole-dwh828chkx9c\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-dwh828chkx9c\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0074eb0c-2b24-41cb-a7ff-604c4e76ad09\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0074eb0c-2b24-41cb-a7ff-604c4e76ad09\",\n      \"name\": \"0074eb0c-2b24-41cb-a7ff-604c4e76ad09\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T16:33:20.010489415Z\",\n      \"endTime\": \"2026-10-13T16:57:21.684870881Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0nldrfocqarmt0r34ht6jph1m9d19 still exists (deletion dispatched at 2026-10-13T16:33:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0074eb0c-2b24-41cb-a7ff-604c4e76ad09\n            correlation ID: d6c08569-5a60-4a0f-bb76-207075ebe9e3\n            client request ID: f4713fe8-8c37-423d-a5df-6a24a2af5b2d\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"669ef81600d88868158f50a61a32eb55792068d452440bf21253695791921ce2","pr_number":7496,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"827a5564727e582e9b07423822965a3f6d9902a3cc857595f1ab3814f76516c1","pr_number":7496,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"cd7a71f4ca6cd6df4fa5f6e8e1c80b538d136ae3f5c114c7ba728953809926e5","pr_number":7496,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"36601573516c72370b5de3f2f5199840f1883a8e5ad5f6d84f833bef7eb8e57a","failure_pattern":"Command failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:\u003chex\u003e arohcpsvcdev.azurecr.io/nshneor/cs:\u003chex\u003e --from-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json --to-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json","search_query":"Command failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d arohcpsvcdev.azurecr.io/nshneor/cs:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d --from-registry-config /tmp/tmp.X2puJ9ooST/containers/auth.json --to-registry-config /tmp/tmp.X2puJ9ooST/containers/auth.json","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/global/mirror-image","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-09-28"],"prior_runs_affected":5,"prior_last_seen_at":"2026-09-29T13:40:16Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/global/mirror-image","occurrences":2}],"full_error_samples":["time=2026-10-16T09:14:54.862Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e clustersService.image.registry \u003cnil\u003e}:{\u003cnil\u003e clustersService.image.repository \u003cnil\u003e}@{\u003cnil\u003e clustersService.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-10-16T09:15:30.118Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nFetch pull secret for source registry arohcpsvcdev from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image arohcpsvcdev/nshneor/cs@sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d to arohcpsvcdev.azurecr.io/nshneor/cs:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d.\\nThe image will still be available under it's original digest sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d in the target registry.\\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d arohcpsvcdev.azurecr.io/nshneor/cs:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d --from-registry-config /tmp/tmp.jIM7uBoqB8/containers/auth.json --to-registry-config /tmp/tmp.jIM7uBoqB8/containers/auth.json\\n exit status 1\"","time=2026-10-15T14:28:12.191Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e clustersService.image.registry \u003cnil\u003e}:{\u003cnil\u003e clustersService.image.repository \u003cnil\u003e}@{\u003cnil\u003e clustersService.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-10-15T14:28:45.415Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nFetch pull secret for source registry arohcpsvcdev from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image arohcpsvcdev/nshneor/cs@sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d to arohcpsvcdev.azurecr.io/nshneor/cs:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d.\\nThe image will still be available under it's original digest sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d in the target registry.\\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d arohcpsvcdev.azurecr.io/nshneor/cs:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d --from-registry-config /tmp/tmp.X2puJ9ooST/containers/auth.json --to-registry-config /tmp/tmp.X2puJ9ooST/containers/auth.json\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7559/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111011832884891648","occurred_at":"2026-10-16T08:30:16Z","signature_id":"062e807a8092da1711f0e05e391e8b306673f16936facbff876328c71872d5ee","pr_number":7559,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7559/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110722472755073024","occurred_at":"2026-10-15T13:20:27Z","signature_id":"062e807a8092da1711f0e05e391e8b306673f16936facbff876328c71872d5ee","pr_number":7559,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,1,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":13,"prior_last_seen_at":"2026-10-09T17:15:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":14,"prior_last_seen_at":"2026-10-09T20:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-mpwdzx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x149c10e309a0\u003e: \n    DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x149c10e30940\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-mpwdzx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x149c10e309a0\u003e: \n    DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x149c10e30940\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-s9x9xg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d3dff18b020\u003e: \n    DNS for route host agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1d3dff18afe0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-s9x9xg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d3dff18b020\u003e: \n    DNS for route host agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1d3dff18afe0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7237/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109549126109630464","occurred_at":"2026-10-12T07:37:59Z","signature_id":"ebfef50e4f33847feb7608db52e9f990adccd34b42505d1aca31667f15925d12","pr_number":7237,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109538389626720256","occurred_at":"2026-10-12T06:55:20Z","signature_id":"4673534f224bf242b8df8885a07c9c85c02ab3a626129d3db6b153550df27ba6","pr_number":7451,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"resource group \"rg-cluster-back-version-4285twt9lhf2\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,1,1,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-28","2026-10-05"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-07T12:59:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x36cea0d80120\u003e: \n    resource group \"rg-cluster-back-version-4285twt9lhf2\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x36ce9f324380\u003e{\n                msg: \"resource group \\\"rg-cluster-back-version-4285twt9lhf2\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x36ce9f324360\u003e{\n                    msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n                    err: \u003c*fmt.wrapErrors | 0x36cea0a86750\u003e{\n                        msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n                        errs: [\n                            \u003c*fmt.wrapErrors | 0x36ce9f315470\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x36ce9f860ef0\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                            \u003c*fmt.wrapErrors | 0x36ce9f315470\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x36ce9f860ef0\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                        ],\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x36cea0d80120\u003e: \n    resource group \"rg-cluster-back-version-4285twt9lhf2\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x36ce9f324380\u003e{\n                msg: \"resource group \\\"rg-cluster-back-version-4285twt9lhf2\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x36ce9f324360\u003e{\n                    msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n                    err: \u003c*fmt.wrapErrors | 0x36cea0a86750\u003e{\n                        msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n                        errs: [\n                            \u003c*fmt.wrapErrors | 0x36ce9f315470\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x36ce9f860ef0\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                            \u003c*fmt.wrapErrors | 0x36ce9f315470\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x36ce9f860ef0\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                        ],\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x151218b082b8\u003e: \n    resource group \"rg-cluster-back-version-9554sthg8hnz\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x151216d07800\u003e{\n                msg: \"resource group \\\"rg-cluster-back-version-9554sthg8hnz\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x151216d077a0\u003e{\n                    msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                    err: \u003c*fmt.wrapErrors | 0x1512170c3950\u003e{\n                        msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                        errs: [\n                            \u003c*fmt.wrapErrors | 0x1512170c3920\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x15121775bc50\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                            \u003c*fmt.wrapErrors | 0x1512170c3920\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x15121775bc50\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                        ],\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x151218b082b8\u003e: \n    resource group \"rg-cluster-back-version-9554sthg8hnz\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x151216d07800\u003e{\n                msg: \"resource group \\\"rg-cluster-back-version-9554sthg8hnz\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x151216d077a0\u003e{\n                    msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                    err: \u003c*fmt.wrapErrors | 0x1512170c3950\u003e{\n                        msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                        errs: [\n                            \u003c*fmt.wrapErrors | 0x1512170c3920\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x15121775bc50\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                            \u003c*fmt.wrapErrors | 0x1512170c3920\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x15121775bc50\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                        ],\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"ecb5ffa542f87571700262dae2b7c84f4536526d903a70c009dcccac42adf516","pr_number":7496,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","signature_id":"b46b21369678c0c798aca134326c4166ed0c77d047a753de7c68744c3f55b704","pr_number":6920,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":12,"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":17,"prior_last_seen_at":"2026-10-05T16:21:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0x15cfa7358820\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-kkjsbcrkrfqs\" to finish creating: GET https://rp.j3670144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d48f5df-4419-42d0-908d-f55c9e58b43d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d48f5df-4419-42d0-908d-f55c9e58b43d\",\n      \"name\": \"3d48f5df-4419-42d0-908d-f55c9e58b43d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T20:00:20.459546459Z\",\n      \"endTime\": \"2026-10-14T20:29:21.27372498Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.37 is Partial (want Completed), started 24m22s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3670144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d48f5df-4419-42d0-908d-f55c9e58b43d\n            correlation ID: 7a9ff375-e54e-4cef-a85f-da37d27bb541\n            client request ID: 20f4c917-baec-45d6-a1e1-7f0ef2326c1c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0x15cfa7358820\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-kkjsbcrkrfqs\" to finish creating: GET https://rp.j3670144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d48f5df-4419-42d0-908d-f55c9e58b43d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d48f5df-4419-42d0-908d-f55c9e58b43d\",\n      \"name\": \"3d48f5df-4419-42d0-908d-f55c9e58b43d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T20:00:20.459546459Z\",\n      \"endTime\": \"2026-10-14T20:29:21.27372498Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.37 is Partial (want Completed), started 24m22s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3670144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d48f5df-4419-42d0-908d-f55c9e58b43d\n            correlation ID: 7a9ff375-e54e-4cef-a85f-da37d27bb541\n            client request ID: 20f4c917-baec-45d6-a1e1-7f0ef2326c1c\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-8fzqsptw7bpr/cluster-candidate-5-0-g7bpwx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x4abdbffea60\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-g7bpwx: failed waiting for cluster=\"cluster-candidate-5-0-g7bpwx\" in resourcegroup=\"rg-candidate-5-0-8fzqsptw7bpr\" to finish creating: GET https://rp.j3579520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b237387d-d51f-41fb-8762-715d9d2b7454\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b237387d-d51f-41fb-8762-715d9d2b7454\",\n      \"name\": \"b237387d-d51f-41fb-8762-715d9d2b7454\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T07:05:39.437908469Z\",\n      \"endTime\": \"2026-10-14T07:34:48.061766511Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.6 is Partial (want Completed), started 12m48s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3579520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b237387d-d51f-41fb-8762-715d9d2b7454\n            correlation ID: 92af7c84-e19d-482a-a1fe-a87a2b603856\n            client request ID: 2dc5545b-5fc8-4471-80a2-89372581489c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-8fzqsptw7bpr/cluster-candidate-5-0-g7bpwx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x4abdbffea60\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-g7bpwx: failed waiting for cluster=\"cluster-candidate-5-0-g7bpwx\" in resourcegroup=\"rg-candidate-5-0-8fzqsptw7bpr\" to finish creating: GET https://rp.j3579520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b237387d-d51f-41fb-8762-715d9d2b7454\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b237387d-d51f-41fb-8762-715d9d2b7454\",\n      \"name\": \"b237387d-d51f-41fb-8762-715d9d2b7454\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T07:05:39.437908469Z\",\n      \"endTime\": \"2026-10-14T07:34:48.061766511Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.6 is Partial (want Completed), started 12m48s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3579520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b237387d-d51f-41fb-8762-715d9d2b7454\n            correlation ID: 92af7c84-e19d-482a-a1fe-a87a2b603856\n            client request ID: 2dc5545b-5fc8-4471-80a2-89372581489c\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110447856593670144","occurred_at":"2026-10-14T19:09:11Z","signature_id":"1d72d1eb80a478cce43c5b77b2461ed1a426d75211fc2d70cddaa21ad3a76436","pr_number":7529,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110251822873579520","occurred_at":"2026-10-14T06:10:15Z","signature_id":"afb8138a4654c4414a50bcc92d291c8bdf260bd4de2e39b80a917c8525fe79d5","pr_number":7504,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9da4ad2c21fc0e265207db4c5abcfdd0631935b407c3edcd41248ee669719f25","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,1,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x315b320b0100\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-mjm476sds2q9\" to finish creating: GET https://rp.j4344192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/be19e499-8b2d-4dc2-9a9d-968d043575f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/be19e499-8b2d-4dc2-9a9d-968d043575f5\",\n      \"name\": \"be19e499-8b2d-4dc2-9a9d-968d043575f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T11:09:27.202594199Z\",\n      \"endTime\": \"2026-10-15T11:38:27.187446227Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-pkg2x-wsc5s: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-mjm476sds2q9--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-pkg2x-wsc5s to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-pkg2x-wsc5s: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-mjm476sds2q9--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-pkg2x-wsc5s to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4344192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/be19e499-8b2d-4dc2-9a9d-968d043575f5\n            correlation ID: 36e13d07-8647-4ed3-96df-c8b4fe4cea15\n            client request ID: 9d12f59b-3610-4a88-aed2-e3b9cf9f6a2e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x315b320b0100\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-mjm476sds2q9\" to finish creating: GET https://rp.j4344192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/be19e499-8b2d-4dc2-9a9d-968d043575f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/be19e499-8b2d-4dc2-9a9d-968d043575f5\",\n      \"name\": \"be19e499-8b2d-4dc2-9a9d-968d043575f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T11:09:27.202594199Z\",\n      \"endTime\": \"2026-10-15T11:38:27.187446227Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-pkg2x-wsc5s: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-mjm476sds2q9--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-pkg2x-wsc5s to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-pkg2x-wsc5s: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-mjm476sds2q9--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-pkg2x-wsc5s to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4344192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/be19e499-8b2d-4dc2-9a9d-968d043575f5\n            correlation ID: 36e13d07-8647-4ed3-96df-c8b4fe4cea15\n            client request ID: 9d12f59b-3610-4a88-aed2-e3b9cf9f6a2e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x361e46553da0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-dx8x9nb92ts2\" to finish creating: GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\",\n      \"name\": \"09be8631-a40d-4148-b3b2-3144216d4fe7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T11:44:57.797440876Z\",\n      \"endTime\": \"2026-10-12T12:14:05.478563828Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-48lvb-qrq6s: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-dx8x9nb92ts2--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-48lvb-qrq6s to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-48lvb-qrq6s: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-dx8x9nb92ts2--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-48lvb-qrq6s to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\n            correlation ID: 06eef5ee-18e5-459b-b472-73166e064f53\n            client request ID: c3ba459a-88fd-41d3-aa0f-d4135fceec1f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x361e46553da0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-dx8x9nb92ts2\" to finish creating: GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\",\n      \"name\": \"09be8631-a40d-4148-b3b2-3144216d4fe7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T11:44:57.797440876Z\",\n      \"endTime\": \"2026-10-12T12:14:05.478563828Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-48lvb-qrq6s: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-dx8x9nb92ts2--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-48lvb-qrq6s to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-48lvb-qrq6s: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-dx8x9nb92ts2--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-48lvb-qrq6s to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\n            correlation ID: 06eef5ee-18e5-459b-b472-73166e064f53\n            client request ID: c3ba459a-88fd-41d3-aa0f-d4135fceec1f\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7522/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110675476174344192","occurred_at":"2026-10-15T10:13:41Z","signature_id":"e5849cc16bf78f707dd78a633c3f41b9b7b33955664677f31027772204df8ab0","pr_number":7522,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7420/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109588388414754816","occurred_at":"2026-10-12T10:14:00Z","signature_id":"6ca522238814ece9529f702529a6bcb452636d575084a8828e41559f858abbbd","pr_number":7420,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"9220c54af3cdcf82280969ee7ca39e91829e564fcd5ac622742ec149542a29fd","failure_pattern":"ERROR CODE: DeploymentFailed; detail code HttpRequestException; detail message Error while processing network security perimeter association: An error occurred while sending the request.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=2)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-24T16:41:24Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=2)","occurrences":2}],"full_error_samples":["time=2026-10-14T19:14:18.297Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 description=\"Step nsp\\n  Kind: ARM\\n  Template: templates/mgmt-nsp.bicep\\n  Parameters: configurations/mgmt-nsp.tmpl.bicepparam\"\ntime=2026-10-14T19:14:18.788Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2\ntime=2026-10-14T19:14:19.241Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 deployment=cf8c9a34812ddee83e87f5e1fcb73992e7f0046ac9c8abee6011c27a19ac7c31 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9264896-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcf8c9a34812ddee83e87f5e1fcb73992e7f0046ac9c8abee6011c27a19ac7c31\ntime=2026-10-14T19:14:49.370Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9264896-mgmt-2/providers/Microsoft.Resources/deployments/cf8c9a34812ddee83e87f5e1fcb73992e7f0046ac9c8abee6011c27a19ac7c31/operationStatuses/08584096016264886235\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9264896-mgmt-2/providers/Microsoft.Resources/deployments/profile-xpjnxfpmcvora\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9264896-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j9264896-mgmt-2/resourceAssociations/nsp-j9264896-mgmt-2-ra-a9c395a5-2fe0-5927-97a4-6856e96269ae\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"HttpRequestException\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9264896-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j9264896-mgmt-2/resourceAssociations/nsp-j9264896-mgmt-2-ra-a9c395a5-2fe0-5927-97a4-6856e96269ae, Error message: An error occurred while sending the request.\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-10-14T15:57:00.478Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 description=\"Step nsp\\n  Kind: ARM\\n  Template: templates/mgmt-nsp.bicep\\n  Parameters: configurations/mgmt-nsp.tmpl.bicepparam\"\ntime=2026-10-14T15:57:01.032Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2\ntime=2026-10-14T15:57:01.677Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 deployment=9e882a024304e709a0349553a748b62ffe4602dc3aa507f98e24a10ee21a2af8 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7217792-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9e882a024304e709a0349553a748b62ffe4602dc3aa507f98e24a10ee21a2af8\ntime=2026-10-14T15:57:32.150Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7217792-mgmt-2/providers/Microsoft.Resources/deployments/9e882a024304e709a0349553a748b62ffe4602dc3aa507f98e24a10ee21a2af8/operationStatuses/08584096134641768649\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7217792-mgmt-2/providers/Microsoft.Resources/deployments/profile-7bczjqgoi25xw\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7217792-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j7217792-mgmt-2/resourceAssociations/nsp-j7217792-mgmt-2-ra-e5d09873-b78f-5e95-bab4-2921347dc438\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"HttpRequestException\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7217792-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j7217792-mgmt-2/resourceAssociations/nsp-j7217792-mgmt-2-ra-e5d09873-b78f-5e95-bab4-2921347dc438, Error message: An error occurred while sending the request.\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110439188909264896","occurred_at":"2026-10-14T18:34:47Z","signature_id":"d465f96a6ebaf98458f369563694ed7b4f0235597cd9af66af75425f966c9780","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110390267277217792","occurred_at":"2026-10-14T15:20:23Z","signature_id":"d465f96a6ebaf98458f369563694ed7b4f0235597cd9af66af75425f966c9780","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"835eba9e18d4ce34b2c45e7a932a8d1b8ec3d4cc9cdfed55b9778db69913e7ed","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RequestTimeout; detail message Database account creation failed. Operation Id: \u003cuuid\u003e, Error : Message: {\"Errors\":[\"The requested operation exceeded maximum alloted time. Learn more: \u003curl\u003e; provider Microsoft.DocumentDB","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,1],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":2}],"full_error_samples":["time=2026-10-18T22:01:09.062Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-18T22:01:10.583Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-10-18T22:01:11.299Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d2f2367984b48d226a7d218c1b893f5e8345defcea4da99b6318b3f53d90ed49 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6019200%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd2f2367984b48d226a7d218c1b893f5e8345defcea4da99b6318b3f53d90ed49\ntime=2026-10-18T22:08:43.317Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6019200/providers/Microsoft.Resources/deployments/d2f2367984b48d226a7d218c1b893f5e8345defcea4da99b6318b3f53d90ed49/operationStatuses/08584092460146839094\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6019200/providers/Microsoft.Resources/deployments/rp-cosmos-account\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6019200/providers/Microsoft.DocumentDB/databaseAccounts/arohcpci01-rp-j6019200\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"RequestTimeout\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Database account creation failed. Operation Id: 29cb1d84-68fc-41bc-85ce-f7a058c770ca, Error : Message: {\\\\\\\\\\\\\\\"Errors\\\\\\\\\\\\\\\":[\\\\\\\\\\\\\\\"The requested operation exceeded maximum alloted time. Learn more: https://aka.ms/cosmosdb-tsg-service-request-timeout\\\\\\\\\\\\\\\"]}\\\\\\\\r\\\\\\\\nActivityId: 510ce633-a87c-4347-9741-f3200e8ace7f, Request URI: /apps/afdfb817-7910-4985-b158-590770c1c1bf/services/f1fdfe14-2eef-4dca-b710-c2c6e8884c3f/partitions/918f52c2-0d35-4150-826f-7edd51cc07ef/replicas/134367860023629100s, RequestStats: \\\\\\\\r\\\\\\\\nRequestStartTime: 2026-10-18T22:07:00.4462358Z, RequestEndTime: 2026-10-18T22:07:05.4461411Z,  Number of regions attempted:1\\\\\\\\r\\\\\\\\n{\\\\\\\\\\\\\\\"systemHistory\\\\\\\\\\\\\\\":[{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-18T22:06:12.0209020Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.273,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":20915668.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0851,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32764,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-18T22:06:22.0359077Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.546,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":20914336.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0463,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-18T22:06:32.0513471Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.449,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":20922484.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0842,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-18T22:06:42.0523437Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.762,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":20935348.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.083,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-18T22:06:52.0699478Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.390,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":20926436.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0518,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-18T22:07:02.0711735Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.332,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":20925148.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0848,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32}]}\\\\\\\\r\\\\\\\\nRequestStart: 2026-10-18T22:07:00.4462358Z; ResponseTime: 2026-10-18T22:07:05.4461411Z; StoreResult: StorePhysicalAddress: rntbd://cdb-ms-prod-centralus1-be424.documents.azure.com:11907/apps/afdfb817-7910-4985-b158-590770c1c1bf/services/f1fdfe14-2eef-4dca-b710-c2c6e8884c3f/partitions/918f52c2-0d35-4150-826f-7edd51cc07ef/replicas/134367860023629100s, LSN: -1, GlobalCommittedLsn: -1, PartitionKeyRangeId: , IsValid: False, StatusCode: 408, SubStatusCode: 0, RequestCharge: 0, ItemLSN: -1, SessionToken: , UsingLocalLSN: False, TransportException: null, BELatencyMs: 4996.861, ActivityId: 510ce633-a87c-4347-9741-f3200e8ace7f, RetryAfterInMs: , globalNRegionCommittedGLSN: -1, ReplicaHealthStatuses: [(port: 11907 | status: Connected | lkt: 10/18/2026 10:07:05 PM)], TransportRequestTimeline: {\\\\\\\\\\\\\\\"requestTimeline\\\\\\\\\\\\\\\":[{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Created\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-18T22:07:00.4462358Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0024},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ChannelAcquisitionStarted\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-18T22:07:00.4462382Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0078},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Pipelined\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-18T22:07:00.4462460Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0881},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Transit Time\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-18T22:07:00.4463341Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 4997.4559},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Received\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-18T22:07:05.4437900Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0577},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Completed\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-18T22:07:05.4438477Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0}],\\\\\\\\\\\\\\\"serviceEndpointStats\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"inflightRequests\\\\\\\\\\\\\\\":1,\\\\\\\\\\\\\\\"openConnections\\\\\\\\\\\\\\\":1},\\\\\\\\\\\\\\\"connectionStats\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"waitforConnectionInit\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"callsPendingReceive\\\\\\\\\\\\\\\":0,\\\\\\\\\\\\\\\"lastSendAttempt\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-18T22:04:46.4616163Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"lastSend\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-18T22:04:46.4616163Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"lastReceive\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-18T22:04:52.4460367Z\\\\\\\\\\\\\\\"},\\\\\\\\\\\\\\\"requestSizeInBytes\\\\\\\\\\\\\\\":867,\\\\\\\\\\\\\\\"responseMetadataSizeInBytes\\\\\\\\\\\\\\\":30,\\\\\\\\\\\\\\\"responseBodySizeInBytes\\\\\\\\\\\\\\\":133};\\\\\\\\r\\\\\\\\n ResourceType: Replica, OperationType: ForceConfigRefresh\\\\\\\\r\\\\\\\\n, SDK: Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-10-12T23:56:30.391Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-12T23:56:32.064Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-10-12T23:56:32.955Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=2f9fa65a0ba704015b11015a72cc4e4b4675fce22bd699e4c355e7650345c6e3 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3444352%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F2f9fa65a0ba704015b11015a72cc4e4b4675fce22bd699e4c355e7650345c6e3\ntime=2026-10-13T00:03:04.471Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3444352/providers/Microsoft.Resources/deployments/2f9fa65a0ba704015b11015a72cc4e4b4675fce22bd699e4c355e7650345c6e3/operationStatuses/08584097574930206657\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3444352/providers/Microsoft.Resources/deployments/rp-cosmos-account\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3444352/providers/Microsoft.DocumentDB/databaseAccounts/arohcpci01-rp-j3444352\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"RequestTimeout\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Database account creation failed. Operation Id: 42664daf-4e8e-4916-9d75-23b7169468da, Error : Message: {\\\\\\\\\\\\\\\"Errors\\\\\\\\\\\\\\\":[\\\\\\\\\\\\\\\"The requested operation exceeded maximum alloted time. Learn more: https://aka.ms/cosmosdb-tsg-service-request-timeout\\\\\\\\\\\\\\\"]}\\\\\\\\r\\\\\\\\nActivityId: 26eaccdb-de17-4cbf-afc1-0c865cfbb1a2, Request URI: /apps/de70b74f-b005-4de6-b2c3-b39046bf9084/services/74353989-92b9-4a77-a49a-b47600be8b14/partitions/a40f5539-0cc6-43c1-b5bc-fdb8dcb67485/replicas/134362021132042649s, RequestStats: \\\\\\\\r\\\\\\\\nRequestStartTime: 2026-10-13T00:02:20.9415661Z, RequestEndTime: 2026-10-13T00:02:26.9109610Z,  Number of regions attempted:1\\\\\\\\r\\\\\\\\n{\\\\\\\\\\\\\\\"systemHistory\\\\\\\\\\\\\\\":[{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:01:28.0336227Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.273,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21877192.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.1501,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":48},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:01:38.0493247Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":1.521,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21840180.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0747,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":48},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:01:48.0505987Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.332,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21844420.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.1332,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":48},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:01:58.0664654Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.975,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21830384.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.1746,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":48},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:02:08.0821422Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.332,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21833552.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0846,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":48},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:02:18.0977872Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":2.281,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21805124.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.1832,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":51}]}\\\\\\\\r\\\\\\\\nRequestStart: 2026-10-13T00:02:20.9415661Z; ResponseTime: 2026-10-13T00:02:26.9109610Z; StoreResult: StorePhysicalAddress: rntbd://cdb-ms-prod-centralus1-be424.documents.azure.com:11306/apps/de70b74f-b005-4de6-b2c3-b39046bf9084/services/74353989-92b9-4a77-a49a-b47600be8b14/partitions/a40f5539-0cc6-43c1-b5bc-fdb8dcb67485/replicas/134362021132042649s, LSN: -1, GlobalCommittedLsn: -1, PartitionKeyRangeId: , IsValid: False, StatusCode: 408, SubStatusCode: 0, RequestCharge: 0, ItemLSN: -1, SessionToken: , UsingLocalLSN: False, TransportException: null, BELatencyMs: 5975.386, ActivityId: 26eaccdb-de17-4cbf-afc1-0c865cfbb1a2, RetryAfterInMs: , globalNRegionCommittedGLSN: -1, ReplicaHealthStatuses: [(port: 11306 | status: Connected | lkt: 10/13/2026 12:02:26 AM)], TransportRequestTimeline: {\\\\\\\\\\\\\\\"requestTimeline\\\\\\\\\\\\\\\":[{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Created\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-13T00:02:20.9415661Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0013},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ChannelAcquisitionStarted\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-13T00:02:20.9415674Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.006},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Pipelined\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-13T00:02:20.9415734Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0628},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Transit Time\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-13T00:02:20.9416362Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 5976.0734},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Received\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-13T00:02:26.9177096Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.1891},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Completed\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-13T00:02:26.9178987Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0}],\\\\\\\\\\\\\\\"serviceEndpointStats\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"inflightRequests\\\\\\\\\\\\\\\":1,\\\\\\\\\\\\\\\"openConnections\\\\\\\\\\\\\\\":1},\\\\\\\\\\\\\\\"connectionStats\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"waitforConnectionInit\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"callsPendingReceive\\\\\\\\\\\\\\\":0,\\\\\\\\\\\\\\\"lastSendAttempt\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:00:07.6413599Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"lastSend\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:00:07.6413599Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"lastReceive\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:00:12.9085149Z\\\\\\\\\\\\\\\"},\\\\\\\\\\\\\\\"requestSizeInBytes\\\\\\\\\\\\\\\":867,\\\\\\\\\\\\\\\"responseMetadataSizeInBytes\\\\\\\\\\\\\\\":30,\\\\\\\\\\\\\\\"responseBodySizeInBytes\\\\\\\\\\\\\\\":133};\\\\\\\\r\\\\\\\\n ResourceType: Replica, OperationType: ForceConfigRefresh\\\\\\\\r\\\\\\\\n, SDK: Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111935962996019200","occurred_at":"2026-10-18T21:42:26Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":7592,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7483/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109790061913444352","occurred_at":"2026-10-12T23:35:23Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":7483,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"255883e0e1738fd02f2daa41d490a5ba122bcfd657ae1579f6fbfb04d4dff13d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=2)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,2,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":9,"prior_last_seen_at":"2026-10-01T19:26:17Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=2)","occurrences":2}],"full_error_samples":["time=2026-10-14T20:02:04.703Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 description=\"Step infra\\n  Kind: ARM\\n  Template: templates/mgmt-infra.bicep\\n  Parameters: configurations/mgmt-infra.tmpl.bicepparam\"\ntime=2026-10-14T20:02:05.214Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2\ntime=2026-10-14T20:02:05.894Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 deployment=c992d66eb9a38cb8bacf66c4633734326584f1d12bbb6c98da67de8819d3eb79 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0582272-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc992d66eb9a38cb8bacf66c4633734326584f1d12bbb6c98da67de8819d3eb79\ntime=2026-10-14T20:03:06.110Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0582272-mgmt-2/providers/Microsoft.Resources/deployments/c992d66eb9a38cb8bacf66c4633734326584f1d12bbb6c98da67de8819d3eb79/operationStatuses/08584095987599895937\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0582272-mgmt-2/providers/Microsoft.Resources/deployments/mgmt-kv-2awbxe73l63vi\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"VaultAlreadyExists\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The vault name 'ah-ci01-mg-j0582272-2' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0582272-mgmt-2/providers/Microsoft.Resources/deployments/msi-kv-g2p3wnorghzau\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"VaultAlreadyExists\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The vault name 'ah-ci01-mi-j0582272-2' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0582272-mgmt-2/providers/Microsoft.Resources/deployments/ah-ci01-me-j0582272-2\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"VaultAlreadyExists\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The vault name 'ah-ci01-me-j0582272-2' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-10-13T10:18:54.749Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 description=\"Step infra\\n  Kind: ARM\\n  Template: templates/mgmt-infra.bicep\\n  Parameters: configurations/mgmt-infra.tmpl.bicepparam\"\ntime=2026-10-13T10:18:55.539Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2\ntime=2026-10-13T10:18:56.816Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 deployment=fa5bd4309de472f0310387105d9534c5f4fd97370ffb7825457798baaa1e4cb1 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0355584-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ffa5bd4309de472f0310387105d9534c5f4fd97370ffb7825457798baaa1e4cb1\ntime=2026-10-13T10:20:27.529Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0355584-mgmt-2/providers/Microsoft.Resources/deployments/fa5bd4309de472f0310387105d9534c5f4fd97370ffb7825457798baaa1e4cb1/operationStatuses/08584097201492563595\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0355584-mgmt-2/providers/Microsoft.Resources/deployments/ah-ci01-me-j0355584-2\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"VaultAlreadyExists\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The vault name 'ah-ci01-me-j0355584-2' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452210310582272","occurred_at":"2026-10-14T19:26:29Z","signature_id":"d517e51fc5b09c819903bacd247416bcbbe167939922787f3a2438a125b3fdc2","pr_number":7492,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109943465260355584","occurred_at":"2026-10-13T09:44:57Z","signature_id":"d517e51fc5b09c819903bacd247416bcbbe167939922787f3a2438a125b3fdc2","pr_number":7440,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,1,1,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-10-05"],"prior_runs_affected":10,"prior_last_seen_at":"2026-10-09T11:02:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x2c1d48a96060\u003e: \n    resource group \"disabled-image-registry-7ddljxcvwjcm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7ddljxcvwjcm\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\",\n      \"name\": \"bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T16:38:07.060491044Z\",\n      \"endTime\": \"2026-10-13T17:02:11.454465514Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\n            correlation ID: bc63c490-e9f1-483e-a3e0-dbf6c6ddf72f\n            client request ID: 93e5d8b4-b42f-4292-ab79-f0ec030e5a84\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x2c1d48a96060\u003e: \n    resource group \"disabled-image-registry-7ddljxcvwjcm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7ddljxcvwjcm\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\",\n      \"name\": \"bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T16:38:07.060491044Z\",\n      \"endTime\": \"2026-10-13T17:02:11.454465514Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\n            correlation ID: bc63c490-e9f1-483e-a3e0-dbf6c6ddf72f\n            client request ID: 93e5d8b4-b42f-4292-ab79-f0ec030e5a84\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x643438c6138\u003e: \n    resource group \"des-encrypt-mfwpss6hqvsd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-mfwpss6hqvsd\" to finish deleting: GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\",\n      \"name\": \"a58b36b6-30fb-4aee-839d-12ae7b447234\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T16:03:15.661006285Z\",\n      \"endTime\": \"2026-10-12T16:27:21.295309747Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\n            correlation ID: 0827ef63-2c1b-4cad-8799-bd4b41067d42\n            client request ID: 2c486539-fc98-4112-bf4c-2bdff7d36160\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x643438c6138\u003e: \n    resource group \"des-encrypt-mfwpss6hqvsd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-mfwpss6hqvsd\" to finish deleting: GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\",\n      \"name\": \"a58b36b6-30fb-4aee-839d-12ae7b447234\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T16:03:15.661006285Z\",\n      \"endTime\": \"2026-10-12T16:27:21.295309747Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\n            correlation ID: 0827ef63-2c1b-4cad-8799-bd4b41067d42\n            client request ID: 2c486539-fc98-4112-bf4c-2bdff7d36160\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"a73e237dc986d50343fffd30aca449541788309f42293d44e9183312dbe9d419","pr_number":7496,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","signature_id":"4ea81d2e607f663f33592f25d4328ddf786e7ed343f4f03175a81981d5aeb666","pr_number":6920,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"de36b52ee87d849c34b3bf8b0df9a50c73bc24fcb413c756452123c1118a8064","failure_pattern":"There is no failure as the matcher passed to Eventually succeeded on its most recent iteration","search_query":"There is no failure as the matcher passed to Eventually succeeded on its most recent iteration","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:394]: Timed out after 90.003s.\ndelegation and authoritative NS answers should remain consistent for at least 30 seconds before issuance\nThere is no failure as the matcher passed to Eventually succeeded on its most recent iteration\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:394]: Timed out after 90.003s.\ndelegation and authoritative NS answers should remain consistent for at least 30 seconds before issuance\nThere is no failure as the matcher passed to Eventually succeeded on its most recent iteration","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:394]: Timed out after 90.000s.\ndelegation and authoritative NS answers should remain consistent for at least 30 seconds before issuance\nThere is no failure as the matcher passed to Eventually succeeded on its most recent iteration\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:394]: Timed out after 90.000s.\ndelegation and authoritative NS answers should remain consistent for at least 30 seconds before issuance\nThere is no failure as the matcher passed to Eventually succeeded on its most recent iteration"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7544/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110520946510008320","occurred_at":"2026-10-14T23:59:39Z","signature_id":"19450299063bae600bd538ea8d3934944e65d651ee47d4e5a838f879f17c9128","pr_number":7544,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110389077940375552","occurred_at":"2026-10-14T15:15:39Z","signature_id":"7eeadc6a06fe33387af72d93bce8e84c520e130092931b2582b514a2cfa0eeda","pr_number":7492,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,1,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":10,"prior_last_seen_at":"2026-10-08T07:57:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x8382000a060\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x8381fe06080\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x8381fe064a0\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x8381fe06480\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x8382000a060\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x8381fe06080\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x8381fe064a0\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x8381fe06480\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x18ba92cf6f8\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x18ba9177500\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x18ba9176100\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x18ba91760e0\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x18ba92cf6f8\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x18ba9177500\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x18ba9176100\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x18ba91760e0\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7528/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110438573613256704","occurred_at":"2026-10-14T18:32:20Z","signature_id":"ad373b5f327b0c9377a01e8a308f067975dee30f288b975f964b13f502d4e9ed","pr_number":7528,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109797931459547136","occurred_at":"2026-10-13T00:06:39Z","signature_id":"ea8a7a2dfe743c69bf60d5e3342f070fda004e14eff8a761750c50c706798fa0","pr_number":6920,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bff205290781819ab19bf99f50804120a81da4620fe1fb58b29673eae7587c8f","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T04:13:54.2670610Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\", ActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", SubActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a \u003e GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\"}, IsPermanent:false})","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,1,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c33df13c000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T04:13:54.2670610Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\", ActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", SubActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a \u003e GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T04:13:54.2670610Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\\\", ActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", SubActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a \u003e GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x3c33df13c540\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T04:13:54.2670610Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\\\", ActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", SubActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a \u003e GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x3c33df6e8000\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-15T04:13:54.2670610Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\",\n                        ActivityId: \"11bf0618-c400-4197-b3c0-0589df8f658a\",\n                        SubActivityId: \"11bf0618-c400-4197-b3c0-0589df8f658a\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"11bf0618-c400-4197-b3c0-0589df8f658a\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a \u003e GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c33df13c000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T04:13:54.2670610Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\", ActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", SubActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"11bf0618-c400-4197-b3c0-0589df8f658a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a \u003e GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-k87btf4j5n7b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T04:13:54.2670610Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\\\", ActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", SubActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a \u003e GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x3c33df13c540\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T04:13:54.2670610Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\\\", ActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", SubActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"11bf0618-c400-4197-b3c0-0589df8f658a\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a \u003e GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x3c33df6e8000\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-15T04:13:54.2670610Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e\",\n                        ActivityId: \"11bf0618-c400-4197-b3c0-0589df8f658a\",\n                        SubActivityId: \"11bf0618-c400-4197-b3c0-0589df8f658a\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"11bf0618-c400-4197-b3c0-0589df8f658a\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;80609e2b-e937-4bfd-abab-066781c9975e ARID=11bf0618-c400-4197-b3c0-0589df8f658a \u003e GW.Http.CallContext/11bf0618-c400-4197-b3c0-0589df8f658a)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tgr8grhvqs5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a8ac5b76060\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tgr8grhvqs5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T08:31:36.3290647Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\", ActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", SubActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 \u003e GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tgr8grhvqs5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T08:31:36.3290647Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\\\", ActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", SubActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 \u003e GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x2a8ac5f001c0\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T08:31:36.3290647Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\\\", ActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", SubActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 \u003e GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x2a8ac67f4100\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-14T08:31:36.3290647Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\",\n                        ActivityId: \"aec379bd-7447-4ac4-a50b-420184492383\",\n                        SubActivityId: \"aec379bd-7447-4ac4-a50b-420184492383\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"aec379bd-7447-4ac4-a50b-420184492383\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 \u003e GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tgr8grhvqs5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a8ac5b76060\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tgr8grhvqs5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T08:31:36.3290647Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\", ActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", SubActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 \u003e GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tgr8grhvqs5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T08:31:36.3290647Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\\\", ActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", SubActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 \u003e GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x2a8ac5f001c0\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T08:31:36.3290647Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\\\", ActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", SubActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"aec379bd-7447-4ac4-a50b-420184492383\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 \u003e GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x2a8ac67f4100\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-14T08:31:36.3290647Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\",\n                        ActivityId: \"aec379bd-7447-4ac4-a50b-420184492383\",\n                        SubActivityId: \"aec379bd-7447-4ac4-a50b-420184492383\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"aec379bd-7447-4ac4-a50b-420184492383\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 \u003e GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558411769778176","occurred_at":"2026-10-15T02:28:32Z","signature_id":"770ac06f7acb59154eca862595eb4bf4400b3c61cdce89dc281592eb5542d356","pr_number":7342,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110265029910597632","occurred_at":"2026-10-14T07:02:43Z","signature_id":"c88d750e35c7c0d4748d9eea15750ad38b0e7332be0d2e329a4d176a4e356283","pr_number":7516,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":9,"prior_last_seen_at":"2026-10-07T12:59:41Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":2}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-16T11:31:52Z\n  Ended:       2026-10-16T11:54:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j9203200-svc\", component=\"backend\", name=\"identityroleassignments\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue identityroleassignments has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-16T11:39:48Z\n  Ended:       2026-10-16T11:54:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j9203200-svc\", component=\"backend\", name=\"dataplaneoidcfederation\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue dataplaneoidcfederation has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T12:07:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j9203200-svc\", component=\"backend\", name=\"ensuremanagedresourcegroup\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue ensuremanagedresourcegroup has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-15T17:09:40Z\n  Ended:       2026-10-15T17:15:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j1515264-svc\", component=\"backend\", name=\"operationclusterupdate\", region=\"centralus\", severity=\"warning\"\n  Description: Backend controller workqueue operationclusterupdate has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"3c7900addeb947a4a4e739b2d41a366679056acc6cfdec3d613c346f9cc41807","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110761908901515264","occurred_at":"2026-10-15T15:57:09Z","signature_id":"3f48c3fd99f803d9b47c1fdee2b7d24ff77a822095f8a5cd52a3f1c610ccfbd1","pr_number":7440,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":15,"trend_counts":[0,0,1,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":42,"prior_last_seen_at":"2026-10-09T08:49:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":2}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-14T20:45:07Z\n  Ended:       2026-10-14T20:56:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j2147200-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j2147200-svc).\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-14T21:06:15Z\n  Ended:       2026-10-14T21:18:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j2147200-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j2147200-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-13T20:14:21Z\n  Ended:       2026-10-13T20:56:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j6273152-svc\", component=\"frontend\", method=\"get\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j6273152-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452322852147200","occurred_at":"2026-10-14T19:26:58Z","signature_id":"823d75e8489768f41c2c16693a58fb2d5bd13ee5dfe3292082f19eb7aee114da","pr_number":7497,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110083918836273152","occurred_at":"2026-10-13T19:02:36Z","signature_id":"6f5244fc0fea589d70c2fa4e58a09e522022dcd0de6fff81e3ffbe8c544084ae","pr_number":7505,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":8,"trend_counts":[0,0,1,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":25,"prior_last_seen_at":"2026-10-07T22:28:41Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":2}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T20:43:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5701632-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.167:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5701632/providers/microsoft.monitor/accounts/services-j5701632\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mxqgp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T20:43:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5701632-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.167:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5701632/providers/microsoft.monitor/accounts/services-j5701632\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mxqgp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-15T20:52:22Z\n  Ended:       2026-10-15T20:58:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5701632-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.167:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5701632/providers/microsoft.monitor/accounts/services-j5701632\", namespace=\"klusterlet-2tfee727vhjhqsf7g4e565boos8lkt5s\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mxqgp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tfee727vhjhqsf7g4e565boos8lkt5s/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","alert fired 12 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-13T18:51:01Z\n  Ended:       2026-10-13T18:58:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-13T18:51:02Z\n  Ended:       2026-10-13T18:59:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-13T18:51:02Z\n  Ended:       2026-10-13T18:59:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2fpcp41do03q2ch5ajhtin6o2onk6\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2te2fpcp41do03q2ch5ajhtin6o2onk6/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-10-13T18:51:02Z\n  Ended:       2026-10-13T18:58:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-10-13T18:52:01Z\n  Ended:       2026-10-13T18:59:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-10-13T18:52:01Z\n  Ended:       2026-10-13T18:59:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-10-13T18:52:01Z\n  Ended:       2026-10-13T18:58:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-10-13T18:52:01Z\n  Ended:       2026-10-13T18:58:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-10-13T18:52:01Z\n  Ended:       2026-10-13T18:59:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-10-13T18:52:58Z\n  Ended:       2026-10-13T18:59:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2fuhn0rfvdiv69jgohnlv63fgjjka\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2te2fuhn0rfvdiv69jgohnlv63fgjjka/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-10-13T18:52:58Z\n  Ended:       2026-10-13T18:59:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2fuhn0rfvdiv69jgohnlv63fgjjka\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2te2fuhn0rfvdiv69jgohnlv63fgjjka/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-10-13T18:54:01Z\n  Ended:       2026-10-13T19:01:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2gdnt0p29rhn6u0lod1kmga0s4ejn\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2te2gdnt0p29rhn6u0lod1kmga0s4ejn/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110810477125701632","occurred_at":"2026-10-15T19:10:09Z","signature_id":"99633a8290acffaf67c81d7d9233eeb86c98a01a69abab7c56b073c615f7fe35","pr_number":7560,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"ff049ca30612ae3c5d981f61312a933989a2df4db1c20adc6d1b9ff278e63fd3","pr_number":7248,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c35e869ec847e18ffe6465660e3d583636fa828b2ff0c004e57c6941d2b85e2e","failure_pattern":"alert [svc] MaestroServerNoReadyReplicas fired","search_query":"alert [svc] MaestroServerNoReadyReplicas fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroServerNoReadyReplicas does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroServerNoReadyReplicas does not fire","occurrences":2}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:17:41Z\n  Severity:    Sev2\n  Labels:      alertname=\"MaestroServerNoReadyReplicas\", cluster=\"ci01-j7660416-svc\", component=\"maestro\", container=\"kube-state-metrics\", deployment=\"maestro\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.75:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7660416/providers/microsoft.monitor/accounts/services-j7660416\", namespace=\"maestro\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-llr7w\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"critical\"\n  Description: No maestro-server replicas have been Ready for 10 minutes. Readiness is gated on Postgres connectivity, so maestro is likely unable to reach its database; Clusters Service will get 503 and HCP cluster provisioning will stall.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T16:17:41Z\n  Severity:    Sev2\n  Labels:      alertname=\"MaestroServerNoReadyReplicas\", cluster=\"ci01-j7660416-svc\", component=\"maestro\", container=\"kube-state-metrics\", deployment=\"maestro\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.75:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7660416/providers/microsoft.monitor/accounts/services-j7660416\", namespace=\"maestro\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-llr7w\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"critical\"\n  Description: No maestro-server replicas have been Ready for 10 minutes. Readiness is gated on Postgres connectivity, so maestro is likely unable to reach its database; Clusters Service will get 503 and HCP cluster provisioning will stall.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-13T18:40:06Z\n  Severity:    Sev2\n  Labels:      alertname=\"MaestroServerNoReadyReplicas\", cluster=\"ci01-j4029440-svc\", component=\"maestro\", container=\"kube-state-metrics\", deployment=\"maestro\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4029440/providers/microsoft.monitor/accounts/services-j4029440\", namespace=\"maestro\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-k5lj9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"critical\"\n  Description: No maestro-server replicas have been Ready for 10 minutes. Readiness is gated on Postgres connectivity, so maestro is likely unable to reach its database; Clusters Service will get 503 and HCP cluster provisioning will stall.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-13T18:40:06Z\n  Severity:    Sev2\n  Labels:      alertname=\"MaestroServerNoReadyReplicas\", cluster=\"ci01-j4029440-svc\", component=\"maestro\", container=\"kube-state-metrics\", deployment=\"maestro\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4029440/providers/microsoft.monitor/accounts/services-j4029440\", namespace=\"maestro\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-k5lj9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"critical\"\n  Description: No maestro-server replicas have been Ready for 10 minutes. Readiness is gated on Postgres connectivity, so maestro is likely unable to reach its database; Clusters Service will get 503 and HCP cluster provisioning will stall."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7512/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111107720617660416","occurred_at":"2026-10-16T14:51:17Z","signature_id":"114cc78e60d3d816b196ff9d1e4915479d09d11c74f7d7fb5230273971e76a73","pr_number":7512,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110042774484029440","occurred_at":"2026-10-13T16:19:32Z","signature_id":"61b46f21ba4b01a9a3f1d792838519213b86b988924028497bb2806fe1283487","pr_number":7502,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9a8854d173302655314e0bdcbc8411f9e276ffe0aadf4e25f0e0bad8f8dcb51d","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors1h5m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors1h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors1h5m does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-09T17:15:54Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors1h5m does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T11:54:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyClusterProvisionErrors1h5m\", cluster=\"ci01-j9203200-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9203200/providers/microsoft.monitor/accounts/services-j9203200\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"cluster-provision-errors\"\n  Description: More than 72% of completed cluster create (install) operations on ci01-j9203200-svc failed over the last hour with at least 3 failures, a 14.4x burn of the 95% SLO budget.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T17:20:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyClusterProvisionErrors1h5m\", cluster=\"ci01-j9626368-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9626368/providers/microsoft.monitor/accounts/services-j9626368\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"cluster-provision-errors\"\n  Description: More than 72% of completed cluster create (install) operations on ci01-j9626368-svc failed over the last hour with at least 3 failures, a 14.4x burn of the 95% SLO budget."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"3f4e95f4a56ef2812656081fda4421f50ba31daadd1af199afbcdf17ce2e3431","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"d53a2236830f49d3e673f190f6889ac45bcba408a0f6c004202adfc7dcecbe92","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d11e817743c9b7b223c4f7499ea10417bad7afb08d112eac38001716535808b0","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors6h30m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors6h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors6h30m does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-09T17:15:54Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors6h30m does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T12:19:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyClusterProvisionErrors6h30m\", cluster=\"ci01-j9203200-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9203200/providers/microsoft.monitor/accounts/services-j9203200\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n  Description: More than 30% of completed cluster create (install) operations on ci01-j9203200-svc failed over the last 6 hours (at least 5 completions), a 6x burn of the 95% SLO budget.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T17:06:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyClusterProvisionErrors6h30m\", cluster=\"ci01-j9626368-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9626368/providers/microsoft.monitor/accounts/services-j9626368\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n  Description: More than 30% of completed cluster create (install) operations on ci01-j9626368-svc failed over the last 6 hours (at least 5 completions), a 6x burn of the 95% SLO budget."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"5c8ff5a4b50c3b1243adb8b3603e526fd1b9318ecd34f70e7fdaab4ed1afa718","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"f0a5ca335c5452170de0c7d7ae55e4a8594a4284ebc53b77ca227c096a57881b","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"abb1429e70773e39316bd0948d4a9486cc702f42873918d630fe98006b70e184","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=1)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,1,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-10-05"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-09T16:24:33Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=1)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-15T02:14:27.753Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 description=\"Step mitigations\\n Kind: Helm\\n\"\ntime=2026-10-15T02:14:28.197Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 values=\"map[azureCnsCpuLimits:map[enabled:true] jobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157]\"\ntime=2026-10-15T02:14:28.197Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-15T02:14:28.543Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 namespace=aro-hcp-mitigations\ntime=2026-10-15T02:14:28.648Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 dryRun=false\ntime=2026-10-15T02:14:28.648Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-15T02:14:28.690Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-15T02:20:27.812Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-15T02:20:27.812Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"","time=2026-10-14T18:50:15.283Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 description=\"Step mitigations\\n Kind: Helm\\n\"\ntime=2026-10-14T18:50:15.838Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 values=\"map[azureCnsCpuLimits:map[enabled:true] jobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157]\"\ntime=2026-10-14T18:50:15.838Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1\ntime=2026-10-14T18:50:16.026Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 namespace=aro-hcp-mitigations\ntime=2026-10-14T18:50:16.350Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 dryRun=false\ntime=2026-10-14T18:50:16.350Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1\ntime=2026-10-14T18:50:16.477Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1\ntime=2026-10-14T18:56:25.364Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 err=\"failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-14T18:56:25.364Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110545325142839296","occurred_at":"2026-10-15T01:36:32Z","signature_id":"36808247d658b9b22b59f418949b27fa87b1655933f39638ef6bec47bfa0327e","pr_number":7133,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110431880728809472","occurred_at":"2026-10-14T18:05:44Z","signature_id":"3eea31dd2ba80ae3c26eda9638a5c4961c637f75b7b51fe4771e43332561b111","pr_number":7133,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"96e9adfd4e670ee16cf7899b2fa9e01ef5f2d945f7e2ae1d347a055685469540","failure_pattern":"failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kubelet-ds (stamp=2)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kubelet-ds (stamp=2)","occurrences":2}],"full_error_samples":["time=2026-10-12T10:35:56.708Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 description=\"Step kubelet-ds\\n Kind: Helm\\n\"\ntime=2026-10-12T10:35:57.392Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 values=map[enabled:true]\ntime=2026-10-12T10:35:57.392Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:35:57.601Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 namespace=kubelet-ds\ntime=2026-10-12T10:35:57.906Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 dryRun=false\ntime=2026-10-12T10:35:57.907Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:35:58.041Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:40:59.966Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded\"\ntime=2026-10-12T10:40:59.967Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded\"","time=2026-10-12T10:30:06.815Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 description=\"Step kubelet-ds\\n Kind: Helm\\n\"\ntime=2026-10-12T10:30:07.429Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 values=map[enabled:true]\ntime=2026-10-12T10:30:07.429Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:30:07.671Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 namespace=kubelet-ds\ntime=2026-10-12T10:30:08.147Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 dryRun=false\ntime=2026-10-12T10:30:08.147Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:30:08.275Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:35:10.285Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded\"\ntime=2026-10-12T10:35:10.286Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109581128020004864","occurred_at":"2026-10-12T09:45:06Z","signature_id":"3987053007afb2ecf8bd1bd674af68356fc44474cc553867e494bd040f1ce9f4","pr_number":7439,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109578665191804928","occurred_at":"2026-10-12T09:35:22Z","signature_id":"3987053007afb2ecf8bd1bd674af68356fc44474cc553867e494bd040f1ce9f4","pr_number":7162,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3","failure_pattern":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,1],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","occurrences":2}],"full_error_samples":["time=2026-10-18T21:53:08.675Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n  Kind: IstioUpgrade\\n  AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-18T21:53:10.010Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j3670528-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-18T21:53:10.010Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j3670528-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-18T21:53:10.816Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j3670528-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-18T21:53:10.816Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j3670528-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-18T21:53:10.816Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j3670528-svc\ntime=2026-10-18T22:08:13.794Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\"","time=2026-10-16T23:13:05.250Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n  Kind: IstioUpgrade\\n  AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-16T23:13:06.422Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6132992-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-16T23:13:06.422Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6132992-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-16T23:13:07.473Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6132992-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-16T23:13:07.473Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6132992-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-16T23:13:07.473Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j6132992-svc\ntime=2026-10-16T23:28:10.126Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111930721403670528","occurred_at":"2026-10-18T21:21:36Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7580,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111224357266132992","occurred_at":"2026-10-16T22:34:46Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.4073319755600815,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,2,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-09T21:37:50Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:123]: Timed out after 651.749s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x48c3b8dec90\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[21:33:14.493]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104\"\n    }�[0m\n    �[37m[21:33:23.438]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:21.8572874Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\\\", ActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", SubActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \\u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[21:33:23.438]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 50/289 failed (KQL: let _startTime = datetime(2026-10-15T01:38:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T01:43:14.4931101Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-15T01:38:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T01:43:14.4931101Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[21:33:23.712]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/cosmosContent\"\n    }�[0m\n    �[37m[21:33:23.712]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[21:33:24.208]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 1498,\n      \"files\": 1\n    }�[0m\n    �[37m[21:33:24.208]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/cosmosContent\"\n    }�[0m\n    �[37m[21:33:24.217]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[21:33:40.606]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n      \"commits\": 793\n    }�[0m\n    �[37m[21:33:40.607]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect\"\n    }�[0m\n    �[37m[21:33:40.961]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n      ]\n    }�[0m\n    �[37m[21:33:41.088]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j8700672-mgmt-1\",\n        \"ci01-j8700672-svc\"\n      ]\n    }�[0m\n    �[37m[21:33:41.127]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespaces\": [\n        \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n      ]\n    }�[0m\n    �[37m[21:33:41.127]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespaceCount\": 4\n    }�[0m\n    �[37m[21:33:41.127]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:33:41.317]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:33:41.478]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:33:41.623]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n    }�[0m\n    �[37m[21:33:43.044]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:21.8572874Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\\\", ActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", SubActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \\u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\\\"}, IsPermanent:false})\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104\"\n    }�[0m\n    �[37m[21:33:43.044]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:21.8572874Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\\\", ActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", SubActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \\u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\\\"}, IsPermanent:false})\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[21:33:43.407]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843\"\n    }�[0m\n    �[37m[21:33:49.648]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:47.6716888Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\\\", ActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", SubActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \\u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[21:33:49.648]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 47/289 failed (KQL: let _startTime = datetime(2026-10-15T01:23:43.4070857Z);\\nlet _endTime = datetime(2026-10-15T01:28:43.4070856Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[21:33:49.744]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/cosmosContent\"\n    }�[0m\n    �[37m[21:33:49.744]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[21:33:50.271]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 1498,\n      \"files\": 1\n    }�[0m\n    �[37m[21:33:50.271]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/cosmosContent\"\n    }�[0m\n    �[37m[21:33:50.281]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[21:34:08.647]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n      \"commits\": 793\n    }�[0m\n    �[37m[21:34:08.648]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect\"\n    }�[0m\n    �[37m[21:34:09.212]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n      ]\n    }�[0m\n    �[37m[21:34:09.309]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j8700672-mgmt-1\",\n        \"ci01-j8700672-svc\"\n      ]\n    }�[0m\n    �[37m[21:34:09.470]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespaces\": [\n        \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n      ]\n    }�[0m\n    �[37m[21:34:09.470]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespaceCount\": 4\n    }�[0m\n    �[37m[21:34:09.470]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:34:09.615]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:34:10.183]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:34:10.617]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n    }�[0m\n    �[37m[21:34:12.156]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:47.6716888Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\\\", ActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", SubActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \\u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\\\"}, IsPermanent:false})\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843\"\n    }�[0m\n    �[37m[21:34:12.157]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:47.6716888Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\\\", ActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", SubActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \\u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\\\"}, IsPermanent:false})\"\n    }�[0m\n    \n      query-skip-kubernetes-events: command failed: exit status 1\n    output: �[37m[21:34:12.559]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767\"\n    }�[0m\n    �[37m[21:34:18.220]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:16.3210435Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\\\", ActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", SubActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \\u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[21:34:18.221]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 47/289 failed (KQL: let _startTime = datetime(2026-10-15T01:24:12.5582990Z);\\nlet _endTime = datetime(2026-10-15T01:29:12.5582989Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[21:34:18.221]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/cosmosContent\"\n    }�[0m\n    �[37m[21:34:18.221]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[21:34:18.723]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 1498,\n      \"files\": 1\n    }�[0m\n    �[37m[21:34:18.724]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/cosmosContent\"\n    }�[0m\n    �[37m[21:34:18.734]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[21:34:35.437]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect\"\n    }�[0m\n    �[37m[21:34:35.845]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n      ]\n    }�[0m\n    �[37m[21:34:36.194]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j8700672-mgmt-1\",\n        \"ci01-j8700672-svc\"\n      ]\n    }�[0m\n    �[37m[21:34:36.313]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespaces\": [\n        \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n      ]\n    }�[0m\n    �[37m[21:34:36.313]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespaceCount\": 4\n    }�[0m\n    �[37m[21:34:36.313]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:34:36.544]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:34:36.863]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:34:37.093]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n    }�[0m\n    �[37m[21:34:38.438]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:16.3210435Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\\\", ActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", SubActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \\u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767\"\n    }�[0m\n    �[37m[21:34:38.438]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:16.3210435Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\\\", ActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", SubActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \\u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[21:34:38.772]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904\"\n    }�[0m\n    �[37m[21:34:46.539]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:46.3046137Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\\\", ActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", SubActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \\u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[21:34:46.539]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 45/289 failed (KQL: let _startTime = datetime(2026-10-15T01:14:38.7716298Z);\\nlet _endTime = datetime(2026-10-15T01:19:38.7716297Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[21:34:47.129]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/cosmosContent\"\n    }�[0m\n    �[37m[21:34:47.129]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[21:34:47.633]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 1498,\n      \"files\": 1\n    }�[0m\n    �[37m[21:34:47.633]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/cosmosContent\"\n    }�[0m\n    �[37m[21:34:47.642]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[21:34:59.291]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect\"\n    }�[0m\n    �[37m[21:34:59.911]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n      ]\n    }�[0m\n    �[37m[21:35:00.221]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j8700672-mgmt-1\",\n        \"ci01-j8700672-svc\"\n      ]\n    }�[0m\n    �[37m[21:35:00.360]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespaces\": [\n        \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n      ]\n    }�[0m\n    �[37m[21:35:00.360]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespaceCount\": 4\n    }�[0m\n    �[37m[21:35:00.360]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:35:00.702]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:35:01.111]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:35:01.638]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n    }�[0m\n    �[37m[21:35:02.339]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:46.3046137Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\\\", ActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", SubActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \\u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904\"\n    }�[0m\n    �[37m[21:35:02.340]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:46.3046137Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\\\", ActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", SubActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \\u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n    }�[0m\n    \n      query-skip-hcp-and-events: command failed: exit status 2\n    output: �[37m[21:35:02.599]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-1950984811\"\n    }�[0m\n    runtime/cgo: pthread_create failed: Resource temporarily unavailable\n    SIGABRT: abort\n    PC=0x7f80df60594c m=4 sigcode=18446744073709551610\n    \n    goroutine 0 gp=0x14197f1334a0 m=4 mp=0x14197f21a008 [idle]:\n    runtime: g 0 gp=0x14197f1334a0: unknown pc 0x7f80df60594c\n    stack: frame={sp:0x7f808bffe790, fp:0x0} stack=[0x7f808b7ff1e0,0x7f808bffede0)\n                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n    00007f808bffe690: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6a0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6b0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6c0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6d0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6e0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6f0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe700: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe710: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe720: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe730: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe740: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe750: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe760: 00000000 00000000  00007f80 df612fa9  ........./a.....\n    00007f808bffe770: 00000000 00000000  00000000 00000011  ................\n    00007f808bffe780: 00000000 00000130  00007f80 df60593e  0.......\u003eY`.....\n    00007f808bffe790: 00000000 00000120  00000000 00000000   ...............\n                      ^ SP\n    00007f808bffe7a0: 00010000 00000013  00000000 00000000  ................\n    00007f808bffe7b0: 00000000 00000000  00007f80 df688cff  ..........h.....\n    00007f808bffe7c0: 00007f80 89ffb910  00007f80 df688cb1  ..........h.....\n    00007f808bffe7d0: 00007f80 89ffb640  00007f80 8bffeaa0  @...............\n    00007f808bffe7e0: 00007f80 8bffe90e  00007f80 8bffe90f  ................\n    00007f808bffe7f0: 00007f80 89ffb640  00007f80 df6037f5  @........7`.....\n    00007f808bffe800: 00007f80 84000020  00000000 00000120   ....... .......\n    00007f808bffe810: 00000000 003d0f00  701e1ebb c8982900  ..=......).....p\n    00007f808bffe820: 00007f80 8bfff640  00000000 00000006  @...............\n    00007f808bffe830: 00007f80 84000c90  00007f80 8bffeb60  ........`.......\n    00007f808bffe840: 00001419 7f1334a0  00007f80 df5b8646  .4......F.[.....\n    00007f808bffe850: 00007f80 df775e90  00007f80 df5a27f3  .^w......'Z.....\n    00007f808bffe860: 00000000 00000020  701e1ebb c8982900   ........).....p\n    00007f808bffe870: 00007f80 897fb000  00007f80 df677ac4  .........zg.....\n    00007f808bffe880: 00000000 00000000  00000000 0000000d  ................\n    runtime: g 0 gp=0x14197f1334a0: unknown pc 0x7f80df60594c\n    stack: frame={sp:0x7f808bffe790, fp:0x0} stack=[0x7f808b7ff1e0,0x7f808bffede0)\n                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n    00007f808bffe690: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6a0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6b0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6c0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6d0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6e0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6f0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe700: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe710: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe720: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe730: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe740: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe750: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe760: 00000000 00000000  00007f80 df612fa9  ........./a.....\n    00007f808bffe770: 00000000 00000000  00000000 00000011  ................\n    00007f808bffe780: 00000000 00000130  00007f80 df60593e  0.......\u003eY`.....\n    00007f808bffe790: 00000000 00000120  00000000 00000000   ...............\n                      ^ SP\n    00007f808bffe7a0: 00010000 00000013  00000000 00000000  ................\n    00007f808bffe7b0: 00000000 00000000  00007f80 df688cff  ..........h.....\n    00007f808bffe7c0: 00007f80 89ffb910  00007f80 df688cb1  ..........h.....\n    00007f808bffe7d0: 00007f80 89ffb640  00007f80 8bffeaa0  @...............\n    00007f808bffe7e0: 00007f80 8bffe90e  00007f80 8bffe90f  ................\n    00007f808bffe7f0: 00007f80 89ffb640  00007f80 df6037f5  @........7`.....\n    00007f808bffe800: 00007f80 84000020  00000000 00000120   ....... .......\n    00007f808bffe810: 00000000 003d0f00  701e1ebb c8982900  ..=......).....p\n    00007f808bffe820: 00007f80 8bfff640  00000000 00000006  @...............\n    00007f808bffe830: 00007f80 84000c90  00007f80 8bffeb60  ........`.......\n    00007f808bffe840: 00001419 7f1334a0  00007f80 df5b8646  .4......F.[.....\n    00007f808bffe850: 00007f80 df775e90  00007f80 df5a27f3  .^w......'Z.....\n    00007f808bffe860: 00000000 00000020  701e1ebb c8982900   ........).....p\n    00007f808bffe870: 00007f80 897fb000  00007f80 df677ac4  .........zg.....\n    00007f808bffe880: 00000000 00000000  00000000 0000000d  ................\n    \n    goroutine 1 gp=0x14197f1321e0 m=nil [sync.WaitGroup.Wait]:\n    runtime.gopark(0x14197fe86c00?, 0x2022a4d?, 0x10?, 0x63?, 0x14197fe86c30?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197ff32be8 sp=0x14197ff32bc8 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x14197fc98610, 0x0, 0x1, 0x0, 0x19)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197ff32c50 sp=0x14197ff32be8 pc=0x1f162b2\n    sync.runtime_SemacquireWaitGroup(0x14197fd289c0?, 0xe0?)\n    \t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x14197ff32c88 sp=0x14197ff32c50 pc=0x1f3a66e\n    sync.(*WaitGroup).Wait(0x14197fc98608)\n    \t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x14197ff32cb0 sp=0x14197ff32c88 pc=0x1f4f445\n    golang.org/x/sync/errgroup.(*Group).Wait(0x14197fc98600)\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x14197ff32cd0 sp=0x14197ff32cb0 pc=0x362507e\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x14197f521dc0, {0x59f0b08, 0x14197f530fc0}, {0x14197fcca000, 0x15, 0x1c})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x14197ff32d90 sp=0x14197ff32cd0 pc=0x3927e05\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x14197f521dc0, {0x59f0b08, 0x14197f530fc0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:500 +0x4b08 fp=0x14197ff33968 sp=0x14197ff32d90 pc=0x39261c8\n    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedQueryOptions).RunQuery(0x14197f1401e0, {0x59f0b08, 0x14197f530fc0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_cmd.go:75 +0x565 fp=0x14197ff33b30 sp=0x14197ff33968 pc=0x3937005\n    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawQueryOptions).Run(0x0?, {0x59f0b08, 0x14197f530fc0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_options.go:178 +0x54 fp=0x14197ff33b58 sp=0x14197ff33b30 pc=0x3939d34\n    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryCommand.func1(0x14197f4ba008?, {0x14197f5f82a0?, 0x4?, 0x553b0e2?})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_cmd.go:45 +0x28 fp=0x14197ff33b80 sp=0x14197ff33b58 pc=0x3936a68\n    github.com/spf13/cobra.(*Command).execute(0x14197f4ba008, {0x14197f2c0030, 0xe, 0xf})\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x14197ff33d28 sp=0x14197ff33b80 pc=0x212e614\n    github.com/spf13/cobra.(*Command).ExecuteC(0x14197f625b08)\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x14197ff33e18 sp=0x14197ff33d28 pc=0x212efa5\n    github.com/spf13/cobra.(*Command).Execute(...)\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071\n    github.com/spf13/cobra.(*Command).ExecuteContext(...)\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064\n    main.main()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x14197ff33f48 sp=0x14197ff33e18 pc=0x47ddd6a\n    runtime.main()\n    \t/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x14197ff33fe0 sp=0x14197ff33f48 pc=0x1f01975\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197ff33fe8 sp=0x14197ff33fe0 pc=0x1f40801\n    \n    goroutine 2 gp=0x14197f132d20 m=nil [force gc (idle)]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f2fa8 sp=0x14197f1f2f88 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.forcegchelper()\n    \t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x14197f1f2fe0 sp=0x14197f1f2fa8 pc=0x1f01c93\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f2fe8 sp=0x14197f1f2fe0 pc=0x1f40801\n    created by runtime.init.7 in goroutine 1\n    \t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\n    \n    goroutine 18 gp=0x14197f2843c0 m=nil [GC sweep wait]:\n    runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1ee788 sp=0x14197f1ee768 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.bgsweep(0x14197f292000)\n    \t/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x14197f1ee7c8 sp=0x14197f1ee788 pc=0x1ee9f11\n    runtime.gcenable.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x14197f1ee7e0 sp=0x14197f1ee7c8 pc=0x1edadd7\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1ee7e8 sp=0x14197f1ee7e0 pc=0x1f40801\n    created by runtime.gcenable in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\n    \n    goroutine 19 gp=0x14197f2845a0 m=nil [GC scavenge wait]:\n    runtime.gopark(0x138d6d?, 0x3979fd?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1eef78 sp=0x14197f1eef58 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.(*scavengerState).park(0x82979c0)\n    \t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x14197f1eefa8 sp=0x14197f1eef78 pc=0x1ee77e9\n    runtime.bgscavenge(0x14197f292000)\n    \t/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x14197f1eefc8 sp=0x14197f1eefa8 pc=0x1ee7d99\n    runtime.gcenable.gowrap2()\n    \t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x14197f1eefe0 sp=0x14197f1eefc8 pc=0x1edad97\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1eefe8 sp=0x14197f1eefe0 pc=0x1f40801\n    created by runtime.gcenable in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\n    \n    goroutine 20 gp=0x14197f284780 m=nil [GOMAXPROCS updater (idle)]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1ef788 sp=0x14197f1ef768 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.updateMaxProcsGoroutine()\n    \t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x14197f1ef7e0 sp=0x14197f1ef788 pc=0x1f10267\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1ef7e8 sp=0x14197f1ef7e0 pc=0x1f40801\n    created by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\n    \t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\n    \n    goroutine 21 gp=0x14197f284d20 m=nil [finalizer wait]:\n    runtime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x14197f1f2601?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f2620 sp=0x14197f1f2600 pc=0x1f38d2e\n    runtime.runFinalizers()\n    \t/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x14197f1f27e0 sp=0x14197f1f2620 pc=0x1ed9cc7\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f27e8 sp=0x14197f1f27e0 pc=0x1f40801\n    created by runtime.createfing in goroutine 1\n    \t/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d\n    \n    goroutine 22 gp=0x14197f47c960 m=nil [cleanup wait]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1eff68 sp=0x14197f1eff48 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.(*cleanupQueue).dequeue(0x82980e0)\n    \t/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x14197f1effa0 sp=0x14197f1eff68 pc=0x1ed6074\n    runtime.runCleanups()\n    \t/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x14197f1effe0 sp=0x14197f1effa0 pc=0x1ed66e5\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1effe8 sp=0x14197f1effe0 pc=0x1f40801\n    created by runtime.(*cleanupQueue).createGs in goroutine 1\n    \t/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5\n    \n    goroutine 23 gp=0x14197f5a30e0 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125b862b4ec7?, 0x1?, 0x21?, 0xab?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f0740 sp=0x14197f1f0720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x14197f29d9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f07c8 sp=0x14197f1f0740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f07e0 sp=0x14197f1f07c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f07e8 sp=0x14197f1f07e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 3 gp=0x14197f133680 m=nil [runnable]:\n    runtime.gopark(0x125b8625d223?, 0x3?, 0x89?, 0x19?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f3740 sp=0x14197f1f3720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x14197f29d9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f37c8 sp=0x14197f1f3740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f37e0 sp=0x14197f1f37c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f37e8 sp=0x14197f1f37e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 34 gp=0x14197f604000 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125b862e8e02?, 0x3?, 0xe6?, 0x2d?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60a740 sp=0x14197f60a720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x14197f29d9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f60a7c8 sp=0x14197f60a740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f60a7e0 sp=0x14197f60a7c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60a7e8 sp=0x14197f60a7e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 24 gp=0x14197f5a32c0 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125b862118da?, 0x1?, 0x4d?, 0x9a?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f0f40 sp=0x14197f1f0f20 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x14197f29d9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f0fc8 sp=0x14197f1f0f40 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f0fe0 sp=0x14197f1f0fc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f0fe8 sp=0x14197f1f0fe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 4 gp=0x14197f133860 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125b7fe90140?, 0x3?, 0x20?, 0x3a?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f3f40 sp=0x14197f1f3f20 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x14197f29d9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f3fc8 sp=0x14197f1f3f40 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f3fe0 sp=0x14197f1f3fc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f3fe8 sp=0x14197f1f3fe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 35 gp=0x14197f6041e0 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125b862a34bd?, 0x3?, 0x4?, 0x15?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60af40 sp=0x14197f60af20 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x14197f29d9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f60afc8 sp=0x14197f60af40 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f60afe0 sp=0x14197f60afc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60afe8 sp=0x14197f60afe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 25 gp=0x14197f5a34a0 m=3 mp=0x14197f1a5808 [GC mark termination]:\n    runtime.systemstack_switch()\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x14197f1f16a0 sp=0x14197f1f1690 pc=0x1f3ebc8\n    runtime.forEachP(...)\n    \t/usr/lib/golang/src/runtime/proc.go:2112\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x14197f1f1740 sp=0x14197f1f16a0 pc=0x1edbc34\n    runtime.gcBgMarkWorker(0x14197f29d9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x14197f1f17c8 sp=0x14197f1f1740 pc=0x1edda88\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f17e0 sp=0x14197f1f17c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f17e8 sp=0x14197f1f17e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 5 gp=0x14197f133a40 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125b8624ea9b?, 0x1?, 0xfd?, 0x7a?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f4740 sp=0x14197f1f4720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x14197f29d9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f47c8 sp=0x14197f1f4740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f47e0 sp=0x14197f1f47c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f47e8 sp=0x14197f1f47e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 50 gp=0x14197f768f00 m=nil [select, locked to thread]:\n    runtime.gopark(0x14197f60c7a8?, 0x2?, 0x1?, 0x8f?, 0x14197f60c794?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60c628 sp=0x14197f60c608 pc=0x1f38d2e\n    runtime.selectgo(0x14197f60c7a8, 0x14197f60c790, 0x0?, 0x0, 0x0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f60c758 sp=0x14197f60c628 pc=0x1f152a5\n    runtime.ensureSigM.func1()\n    \t/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x14197f60c7e0 sp=0x14197f60c758 pc=0x1f328e8\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60c7e8 sp=0x14197f60c7e0 pc=0x1f40801\n    created by runtime.ensureSigM in goroutine 1\n    \t/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5\n    \n    goroutine 26 gp=0x14197f3b34a0 m=7 mp=0x14197f55b008 [syscall]:\n    runtime.notetsleepg(0x82bdea0, 0xffffffffffffffff)\n    \t/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x14197f6077a0 sp=0x14197f607778 pc=0x1ece509\n    os/signal.signal_recv()\n    \t/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x14197f6077c0 sp=0x14197f6077a0 pc=0x1f3b298\n    os/signal.loop()\n    \t/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x14197f6077e0 sp=0x14197f6077c0 pc=0x2030553\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f6077e8 sp=0x14197f6077e0 pc=0x1f40801\n    created by os/signal.Notify.func1.1 in goroutine 1\n    \t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\n    \n    goroutine 27 gp=0x14197f6894a0 m=nil [select]:\n    runtime.gopark(0x14197f607f98?, 0x2?, 0x68?, 0x81?, 0x14197f607f8c?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f607e28 sp=0x14197f607e08 pc=0x1f38d2e\n    runtime.selectgo(0x14197f607f98, 0x14197f607f88, 0x0?, 0x0, 0x0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f607f58 sp=0x14197f607e28 pc=0x1f152a5\n    os/signal.NotifyContext.func1()\n    \t/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x14197f607fe0 sp=0x14197f607f58 pc=0x202ff9c\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f607fe8 sp=0x14197f607fe0 pc=0x1f40801\n    created by os/signal.NotifyContext in goroutine 1\n    \t/usr/lib/golang/src/os/signal/signal.go:291 +0x158\n    \n    goroutine 224 gp=0x14197f689680 m=nil [semacquire]:\n    runtime.gopark(0x7f80904adba8?, 0x1eed7e0?, 0xc0?, 0x68?, 0x4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419800b5b58 sp=0x1419800b5b38 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x1419800b5bc0 sp=0x1419800b5b58 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x1419800b5c60 sp=0x1419800b5bc0 pc=0x1edbb12\n    runtime.gcAssistAlloc(0x14197f689680)\n    \t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x1419800b5cf0 sp=0x1419800b5c60 pc=0x1ee06b6\n    runtime.deductAssistCredit(0x7ffce51f9080?)\n    \t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x1419800b5d10 sp=0x1419800b5cf0 pc=0x1ed10a6\n    runtime.mallocgc(0x7fe00, 0x4af32a0, 0x1)\n    \t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x1419800b5d40 sp=0x1419800b5d10 pc=0x1f35e07\n    runtime.makeslice(0x14197fc9d0b0?, 0x141983a1fe00?, 0x20000?)\n    \t/usr/lib/golang/src/runtime/slice.go:117 +0x49 fp=0x1419800b5d68 sp=0x1419800b5d40 pc=0x1f3b489\n    encoding/json.(*Decoder).refill(0x14197f658c80)\n    \t/usr/lib/golang/src/encoding/json/stream.go:161 +0xee fp=0x1419800b5db8 sp=0x1419800b5d68 pc=0x200d48e\n    encoding/json.(*Decoder).readValue(0x14197f658c80)\n    \t/usr/lib/golang/src/encoding/json/stream.go:142 +0x85 fp=0x1419800b5e08 sp=0x1419800b5db8 pc=0x200d125\n    encoding/json.(*Decoder).Decode(0x14197f658c80, {0x4cfb340, 0x14197ffa44c0})\n    \t/usr/lib/golang/src/encoding/json/stream.go:65 +0x75 fp=0x1419800b5e38 sp=0x1419800b5e08 pc=0x200cdf5\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readPrimaryTable(0x14197f8c54f0, 0x14197fb50000)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:264 +0x16e fp=0x1419800b5ec8 sp=0x1419800b5e38 pc=0x38d822e\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x14197f8c54f0)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:174 +0x189 fp=0x1419800b5f30 sp=0x1419800b5ec8 pc=0x38d7b89\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x14197f8c54f0, 0x14198007be90)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x1419800b5fc0 sp=0x1419800b5f30 pc=0x38d788c\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x1419800b5fe0 sp=0x1419800b5fc0 pc=0x38d751b\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1419800b5fe8 sp=0x1419800b5fe0 pc=0x1f40801\n    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 183\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc\n    \n    goroutine 87 gp=0x14197fd3c000 m=nil [sync.WaitGroup.Wait]:\n    runtime.gopark(0x14197fcf9e00?, 0x1ed0734?, 0x20?, 0x94?, 0x7f80917bdf06?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf9da0 sp=0x14197fcf9d80 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x14197f69cc50, 0x0, 0x1, 0x0, 0x19)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fcf9e08 sp=0x14197fcf9da0 pc=0x1f162b2\n    sync.runtime_SemacquireWaitGroup(0x14197f86b2d8?, 0x0?)\n    \t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x14197fcf9e40 sp=0x14197fcf9e08 pc=0x1f3a66e\n    sync.(*WaitGroup).Wait(0x14197f69cc48)\n    \t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x14197fcf9e68 sp=0x14197fcf9e40 pc=0x1f4f445\n    golang.org/x/sync/errgroup.(*Group).Wait(0x14197f69cc40)\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x14197fcf9e88 sp=0x14197fcf9e68 pc=0x362507e\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x14197f521d80, {0x59f0b40, 0x14197f8c4b40}, {0x14197fcca000, 0x15, 0x6f63746669687372?}, 0x14197f1c02a0)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x14197fcf9f00 sp=0x14197fcf9e88 pc=0x392976a\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x14197fcf9f78 sp=0x14197fcf9f00 pc=0x392808e\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197fcf9fe0 sp=0x14197fcf9f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf9fe8 sp=0x14197fcf9fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 77 gp=0x14197fd3c1e0 m=nil [IO wait]:\n    runtime.gopark(0x14197fb3e4a0?, 0x1edb176?, 0x80?, 0xed?, 0xb?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f202808 sp=0x14197f2027e8 pc=0x1f38d2e\n    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n    \t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x14197f202840 sp=0x14197f202808 pc=0x1ef9e57\n    internal/poll.runtime_pollWait(0x7f8098109000, 0x72)\n    \t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x14197f202860 sp=0x14197f202840 pc=0x1f375c5\n    internal/poll.(*pollDesc).wait(0x14197f323a80?, 0x14197f34d300?, 0x0)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x14197f202888 sp=0x14197f202860 pc=0x1fbd167\n    internal/poll.(*pollDesc).waitRead(...)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\n    internal/poll.(*FD).Read(0x14197f323a80, {0x14197f34d300, 0x1300, 0x1300})\n    \t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x14197f202920 sp=0x14197f202888 pc=0x1fbe4ce\n    net.(*netFD).Read(0x14197f323a80, {0x14197f34d300?, 0x14197f34d30a?, 0x5?})\n    \t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x14197f202968 sp=0x14197f202920 pc=0x20798c5\n    net.(*conn).Read(0x14197f1300d8, {0x14197f34d300?, 0x7f809057a868?, 0x7f80df53a6d8?})\n    \t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x14197f2029b0 sp=0x14197f202968 pc=0x208b7a5\n    crypto/tls.(*atLeastReader).Read(0x14197f821a10, {0x14197f34d300?, 0x14197f202a98?, 0x1efcc28?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x14197f2029f8 sp=0x14197f2029b0 pc=0x2222ddb\n    bytes.(*Buffer).ReadFrom(0x14197f8a57a8, {0x59a3680, 0x14197f821a10})\n    \t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x14197f202a50 sp=0x14197f2029f8 pc=0x1fe9258\n    crypto/tls.(*Conn).readFromUntil(0x14197f8a5508, {0x59a32c0, 0x14197f1300d8}, 0x222764c?)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x14197f202a88 sp=0x14197f202a50 pc=0x2222fbe\n    crypto/tls.(*Conn).readRecordOrCCS(0x14197f8a5508, 0x0)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x14197f202c68 sp=0x14197f202a88 pc=0x222009b\n    crypto/tls.(*Conn).readRecord(...)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:588\n    crypto/tls.(*Conn).Read(0x14197f8a5508, {0x14197fa26000, 0x1000, 0x4e55540?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x14197f202cd0 sp=0x14197f202c68 pc=0x22267c5\n    net/http.(*persistConn).Read(0x14197f8032c0, {0x14197fa26000?, 0x59a0ea0?, 0x81260e0?})\n    \t/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x14197f202d30 sp=0x14197f202cd0 pc=0x2319d07\n    bufio.(*Reader).fill(0x14197f29b620)\n    \t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f202d68 sp=0x14197f202d30 pc=0x20a6c63\n    bufio.(*Reader).Peek(0x14197f29b620, 0x1)\n    \t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x14197f202d80 sp=0x14197f202d68 pc=0x20a6df3\n    net/http.(*persistConn).readLoop(0x14197f8032c0)\n    \t/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x14197f202fc8 sp=0x14197f202d80 pc=0x231a812\n    net/http.(*Transport).dialConn.gowrap2()\n    \t/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x14197f202fe0 sp=0x14197f202fc8 pc=0x2319197\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f202fe8 sp=0x14197f202fe0 pc=0x1f40801\n    created by net/http.(*Transport).dialConn in goroutine 9\n    \t/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac\n    \n    goroutine 7 gp=0x14197fe025a0 m=nil [sync.Mutex.Lock]:\n    runtime.gopark(0x14197f9d3c68?, 0x14197f9d3c70?, 0x94?, 0x69?, 0x14197f9d3c78?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f9d3c00 sp=0x14197f9d3be0 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x14197fccaad4, 0x0, 0x3, 0x2, 0x16)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197f9d3c68 sp=0x14197f9d3c00 pc=0x1f162b2\n    internal/sync.runtime_SemacquireMutex(0x1000?, 0xd0?, 0x1ed11a5?)\n    \t/usr/lib/golang/src/runtime/sema.go:95 +0x25 fp=0x14197f9d3ca0 sp=0x14197f9d3c68 pc=0x1f3a545\n    internal/sync.(*Mutex).lockSlow(0x14197fccaad0)\n    \t/usr/lib/golang/src/internal/sync/mutex.go:149 +0x15d fp=0x14197f9d3cf0 sp=0x14197f9d3ca0 pc=0x1f4d31d\n    internal/sync.(*Mutex).Lock(...)\n    \t/usr/lib/golang/src/internal/sync/mutex.go:70\n    sync.(*Mutex).Lock(...)\n    \t/usr/lib/golang/src/sync/mutex.go:46\n    net/http.(*http2clientConnReadLoop).streamByID(0x22babb5?, 0x1b, 0x1)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:10083 +0x5b fp=0x14197f9d3d48 sp=0x14197f9d3cf0 pc=0x22e10db\n    net/http.(*http2clientConnReadLoop).processData(0x14197f9d3fa8, 0x14197fc9d200)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9939 +0x45 fp=0x14197f9d3e18 sp=0x14197f9d3d48 pc=0x22e05e5\n    net/http.(*http2clientConnReadLoop).run(0x14197f9d3fa8)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9584 +0x4b6 fp=0x14197f9d3f60 sp=0x14197f9d3e18 pc=0x22de536\n    net/http.(*http2ClientConn).readLoop(0x14197fccaa80)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x14197f9d3fc8 sp=0x14197f9d3f60 pc=0x22dd532\n    net/http.(*http2Transport).newClientConn.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x14197f9d3fe0 sp=0x14197f9d3fc8 pc=0x22d7057\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f9d3fe8 sp=0x14197f9d3fe0 pc=0x1f40801\n    created by net/http.(*http2Transport).newClientConn in goroutine 6\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5\n    \n    goroutine 78 gp=0x14197fe02960 m=nil [select]:\n    runtime.gopark(0x14197f9f7f48?, 0x2?, 0x0?, 0xb6?, 0x14197f9f7ef4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf2d88 sp=0x14197fcf2d68 pc=0x1f38d2e\n    runtime.selectgo(0x14197fcf2f48, 0x14197f9f7ef0, 0x14197f1cd0c0?, 0x0, 0x14197f89c2a0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197fcf2eb8 sp=0x14197fcf2d88 pc=0x1f152a5\n    net/http.(*persistConn).writeLoop(0x14197f8032c0)\n    \t/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x14197fcf2fc8 sp=0x14197fcf2eb8 pc=0x231c426\n    net/http.(*Transport).dialConn.gowrap3()\n    \t/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x14197fcf2fe0 sp=0x14197fcf2fc8 pc=0x2319157\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf2fe8 sp=0x14197fcf2fe0 pc=0x1f40801\n    created by net/http.(*Transport).dialConn in goroutine 9\n    \t/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe\n    \n    goroutine 163 gp=0x14197f7694a0 m=nil [select]:\n    runtime.gopark(0x14197fdd3f08?, 0x6?, 0xa0?, 0x94?, 0x14197fdd3da4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f205c08 sp=0x14197f205be8 pc=0x1f38d2e\n    runtime.selectgo(0x14197f205f08, 0x14197fdd3d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f205d38 sp=0x14197f205c08 pc=0x1f152a5\n    net/http.(*http2clientStream).writeRequest(0x14197f37f380, 0x14197f49e140, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x14197f205f90 sp=0x14197f205d38 pc=0x22da6a8\n    net/http.(*http2clientStream).doRequest(0x14197f37f380, 0x14197f546100?, 0x14197f609798?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x14197f205fb8 sp=0x14197f205f90 pc=0x22d99d8\n    net/http.(*http2ClientConn).roundTrip.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197f205fe0 sp=0x14197f205fb8 pc=0x22d999f\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f205fe8 sp=0x14197f205fe0 pc=0x1f40801\n    created by net/http.(*http2ClientConn).roundTrip in goroutine 184\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n    \n    goroutine 88 gp=0x14197f769a40 m=nil [semacquire]:\n    runtime.gopark(0x141980119958?, 0x1ed3e05?, 0xe8?, 0xd2?, 0x7f80904ad2e8?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x141980119908 sp=0x1419801198e8 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x141980119970 sp=0x141980119908 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x141980119a10 sp=0x141980119970 pc=0x1edbb12\n    runtime.gcAssistAlloc(0x14197f769a40)\n    \t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x141980119aa0 sp=0x141980119a10 pc=0x1ee06b6\n    runtime.deductAssistCredit(0x141983b00000?)\n    \t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x141980119ac0 sp=0x141980119aa0 pc=0x1ed10a6\n    runtime.mallocgc(0x7edd, 0x0, 0x0)\n    \t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x141980119af0 sp=0x141980119ac0 pc=0x1f35e07\n    runtime.slicebytetostring(0x4e537a0?, 0x141983b00000, 0x7edd)\n    \t/usr/lib/golang/src/runtime/string.go:170 +0x77 fp=0x141980119b20 sp=0x141980119af0 pc=0x1f1e397\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x14197f8c4b40}, 0x14197f1c03f0, 0x14197f531740)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:322 +0x91b fp=0x141980119f48 sp=0x141980119b20 pc=0x3920bfb\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x141980119f78 sp=0x141980119f48 pc=0x392800c\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x141980119fe0 sp=0x141980119f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x141980119fe8 sp=0x141980119fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 110 gp=0x14197fd3cb40 m=nil [sync.Mutex.Lock]:\n    runtime.gopark(0x22b8a73?, 0x14197f7e1000?, 0xb0?, 0x4c?, 0x8160a40?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1c7510 sp=0x14197f1c74f0 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x14197fccaad4, 0x0, 0x3, 0x2, 0x16)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197f1c7578 sp=0x14197f1c7510 pc=0x1f162b2\n    internal/sync.runtime_SemacquireMutex(0x14197f1c75d0?, 0x66?, 0x61f?)\n    \t/usr/lib/golang/src/runtime/sema.go:95 +0x25 fp=0x14197f1c75b0 sp=0x14197f1c7578 pc=0x1f3a545\n    internal/sync.(*Mutex).lockSlow(0x14197fccaad0)\n    \t/usr/lib/golang/src/internal/sync/mutex.go:149 +0x15d fp=0x14197f1c7600 sp=0x14197f1c75b0 pc=0x1f4d31d\n    internal/sync.(*Mutex).Lock(...)\n    \t/usr/lib/golang/src/internal/sync/mutex.go:70\n    sync.(*Mutex).Lock(...)\n    \t/usr/lib/golang/src/sync/mutex.go:46\n    net/http.http2transportResponseBody.Read({0x1f3ec12?}, {0x14198313d000?, 0x1eed797?, 0x14197f1c76b0?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9875 +0x209 fp=0x14197f1c7688 sp=0x14197f1c7600 pc=0x22dff89\n    bufio.(*Reader).fill(0x14197f3a00c0)\n    \t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f1c76c0 sp=0x14197f1c7688 pc=0x20a6c63\n    bufio.(*Reader).ReadByte(0x14197f3a00c0)\n    \t/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x14197f1c76d8 sp=0x14197f1c76c0 pc=0x20a73c7\n    compress/flate.(*decompressor).moreBits(0x141980059908)\n    \t/usr/lib/golang/src/compress/flate/inflate.go:697 +0x24 fp=0x14197f1c7710 sp=0x14197f1c76d8 pc=0x2288824\n    compress/flate.(*decompressor).nextBlock(0x141980059908)\n    \t/usr/lib/golang/src/compress/flate/inflate.go:304 +0x28 fp=0x14197f1c7728 sp=0x14197f1c7710 pc=0x2286ba8\n    compress/flate.(*decompressor).Read(0x141980059908, {0x14198314e000, 0x1000, 0x7f8090ef0f00?})\n    \t/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x14197f1c7758 sp=0x14197f1c7728 pc=0x2286dbb\n    compress/gzip.(*Reader).Read(0x14197f672dc8, {0x14198314e000, 0x1000, 0x1000})\n    \t/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x14197f1c77e8 sp=0x14197f1c7758 pc=0x228c838\n    github.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x410d0fa76c5d7d40?, {0x14198314e000?, 0x7f8090ef0d20?, 0x52ffb40?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x14197f1c7818 sp=0x14197f1c77e8 pc=0x38b33e2\n    bufio.(*Reader).fill(0x14197f3a0120)\n    \t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f1c7850 sp=0x14197f1c7818 pc=0x20a6c63\n    bufio.(*Reader).Peek(0x14197f3a0120, 0x1)\n    \t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x14197f1c7868 sp=0x14197f1c7850 pc=0x20a6df3\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.validateJsonResponse(0x14197f3a0120)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:38 +0x25 fp=0x14197f1c78b8 sp=0x14197f1c7868 pc=0x38d6be5\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.newFrameReader({0x59db110, 0x14197fe1e040}, {0x59f0b40, 0x14197f748140})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:26 +0x188 fp=0x14197f1c7960 sp=0x14197f1c78b8 pc=0x38d6a08\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset({0x59f0bb0?, 0x14197f4797a0?}, {0x59db110, 0x14197fe1e040}, 0x1, 0x3e8, 0x1)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:66 +0x19e fp=0x14197f1c79e0 sp=0x14197f1c7960 pc=0x38d737e\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x14197f521d40, {0x59f0bb0, 0x14197f4797a0}, {0x14197f49a330, 0xb}, 0x14197fe1e700, {0x0?, 0x2?, 0x0?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:167 +0x2b7 fp=0x14197f1c7a78 sp=0x14197f1c79e0 pc=0x38e4f17\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197f7a2600}, 0x14197f1c02a0)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x14197f1c7d60 sp=0x14197f1c7a78 pc=0x391b5c5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x14197f521d80, {0x59f0b40, 0x14197fdd8140}, {0x5a00340, 0x14197fc1e5d0}, 0x14197f1c02a0)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:98 +0x366 fp=0x14197f1c7e88 sp=0x14197f1c7d60 pc=0x3929ec6\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14197f1c7f78 sp=0x14197f1c7e88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197f1c7fe0 sp=0x14197f1c7f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1c7fe8 sp=0x14197f1c7fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 89 gp=0x14197fd82960 m=nil [select]:\n    runtime.gopark(0x1419800b7d00?, 0x2?, 0xd7?, 0xfd?, 0x1419800b7c74?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419800b7ac8 sp=0x1419800b7aa8 pc=0x1f38d2e\n    runtime.selectgo(0x1419800b7d00, 0x1419800b7c70, 0x1419801521c0?, 0x0, 0x14197f5daba0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x1419800b7bf8 sp=0x1419800b7ac8 pc=0x1f152a5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x14197f521d80?, {0x59f0b40, 0x14197f8c4b40}, 0x14197f1c02a0, 0x14197f1c03f0)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x1419800b7f10 sp=0x1419800b7bf8 pc=0x392836e\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x1419800b7f78 sp=0x1419800b7f10 pc=0x3927f3d\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x1419800b7fe0 sp=0x1419800b7f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1419800b7fe8 sp=0x1419800b7fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 175 gp=0x14197fd3d2c0 m=nil [sync.Cond.Wait]:\n    runtime.gopark(0x22c2d5b?, 0x14197fe1c080?, 0x0?, 0xe0?, 0x14197f777000?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14198012b978 sp=0x14198012b958 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    sync.runtime_notifyListWait(0x14197f37f3c8, 0x3c)\n    \t/usr/lib/golang/src/runtime/sema.go:617 +0x1b3 fp=0x14198012b9c8 sp=0x14198012b978 pc=0x1f3a8f3\n    sync.(*Cond).Wait(0x1000?)\n    \t/usr/lib/golang/src/sync/cond.go:71 +0x73 fp=0x14198012ba00 sp=0x14198012b9c8 pc=0x1f4d673\n    net/http.(*http2pipe).Read(0x14197f37f3b0, {0x14197ff7e000, 0x1000, 0x1000})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:3928 +0xdb fp=0x14198012ba68 sp=0x14198012ba00 pc=0x22c2c1b\n    net/http.http2transportResponseBody.Read({0x4a54?}, {0x14197ff7e000?, 0x14198012bb80?, 0x2287949?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9852 +0x59 fp=0x14198012baf0 sp=0x14198012ba68 pc=0x22dfdd9\n    bufio.(*Reader).fill(0x14197fcaa000)\n    \t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14198012bb28 sp=0x14198012baf0 pc=0x20a6c63\n    bufio.(*Reader).ReadByte(0x14197fcaa000)\n    \t/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x14198012bb40 sp=0x14198012bb28 pc=0x20a73c7\n    compress/flate.(*decompressor).moreBits(0x141980056008)\n    \t/usr/lib/golang/src/compress/flate/inflate.go:697 +0x24 fp=0x14198012bb78 sp=0x14198012bb40 pc=0x2288824\n    compress/flate.(*decompressor).nextBlock(0x141980056008)\n    \t/usr/lib/golang/src/compress/flate/inflate.go:304 +0x28 fp=0x14198012bb90 sp=0x14198012bb78 pc=0x2286ba8\n    compress/flate.(*decompressor).Read(0x141980056008, {0x141980034b13, 0x4ed, 0xb13?})\n    \t/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x14198012bbc0 sp=0x14198012bb90 pc=0x2286dbb\n    compress/gzip.(*Reader).Read(0x141980000b08, {0x141980034b13, 0x4ed, 0x4ed})\n    \t/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x14198012bc50 sp=0x14198012bbc0 pc=0x228c838\n    github.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x14198012bcf0?, {0x141980034b13?, 0x1000?, 0x0?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x14198012bc80 sp=0x14198012bc50 pc=0x38b33e2\n    bufio.(*Reader).fill(0x141980807010)\n    \t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14198012bcb8 sp=0x14198012bc80 pc=0x20a6c63\n    bufio.(*Reader).ReadSlice(0x141980807010, 0xa)\n    \t/usr/lib/golang/src/bufio/bufio.go:377 +0x29 fp=0x14198012bd00 sp=0x14198012bcb8 pc=0x20a7769\n    bufio.(*Reader).collectFragments(0x141980807010, 0xa)\n    \t/usr/lib/golang/src/bufio/bufio.go:452 +0x85 fp=0x14198012bde0 sp=0x14198012bd00 pc=0x20a7b85\n    bufio.(*Reader).ReadBytes(0x4bbc5a0?, 0xa0?)\n    \t/usr/lib/golang/src/bufio/bufio.go:479 +0x17 fp=0x14198012be60 sp=0x14198012bde0 pc=0x20a7e77\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.(*frameReader).advance(0x141980807000)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:64 +0x45 fp=0x14198012beb0 sp=0x14198012be60 pc=0x38d6d45\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readRoutine(0x141980807000, 0x14197f8c8370)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:85 +0x51 fp=0x14198012bfc0 sp=0x14198012beb0 pc=0x38d7591\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap2()\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x1b fp=0x14198012bfe0 sp=0x14198012bfc0 pc=0x38d74db\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14198012bfe8 sp=0x14198012bfe0 pc=0x1f40801\n    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 184\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x24a\n    \n    goroutine 183 gp=0x14197fd3d860 m=nil [semacquire]:\n    runtime.gopark(0x7f8090f36c08?, 0x1eed7e0?, 0x80?, 0x43?, 0x2?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fac5208 sp=0x14197fac51e8 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fac5270 sp=0x14197fac5208 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x14197fac5310 sp=0x14197fac5270 pc=0x1edbb12\n    runtime.gcAssistAlloc(0x14197fd3d860)\n    \t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x14197fac53a0 sp=0x14197fac5310 pc=0x1ee06b6\n    runtime.deductAssistCredit(0x14197fac53f0?)\n    \t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x14197fac53c0 sp=0x14197fac53a0 pc=0x1ed10a6\n    runtime.mallocgc(0x5500, 0x0, 0x0)\n    \t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x14197fac53f0 sp=0x14197fac53c0 pc=0x1f35e07\n    runtime.growslice(0x141983adc000, 0x14197fac54a8?, 0x1ed3be5?, 0x82bed80?, 0x1ed0417?)\n    \t/usr/lib/golang/src/runtime/slice.go:265 +0x573 fp=0x14197fac5460 sp=0x14197fac53f0 pc=0x1f3ba93\n    strings.(*Builder).Write(0x14197f569ea0, {0x1419839fd000, 0x1000, 0x1000000010765?})\n    \t/usr/lib/golang/src/strings/builder.go:89 +0x90 fp=0x14197fac54b0 sp=0x14197fac5460 pc=0x1ff0cb0\n    bufio.(*Writer).Flush(0x14197fac5698)\n    \t/usr/lib/golang/src/bufio/bufio.go:642 +0x52 fp=0x14197fac54f8 sp=0x14197fac54b0 pc=0x20a8832\n    bufio.(*Writer).WriteString(0x14197fac5698, {0x553a4f6?, 0x553a9e6?})\n    \t/usr/lib/golang/src/bufio/bufio.go:766 +0x165 fp=0x14197fac5568 sp=0x14197fac54f8 pc=0x20a8f65\n    encoding/csv.(*Writer).Write(0x14197fac5688, {0x14198384d480?, 0x4, 0x141983909500?})\n    \t/usr/lib/golang/src/encoding/csv/writer.go:92 +0x2b4 fp=0x14197fac55d0 sp=0x14197fac5568 pc=0x20ad3d4\n    github.com/Azure/azure-kusto-go/azkustodata/query.(*row).String(0x5e?)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/row_impl.go:101 +0x21f fp=0x14197fac56e8 sp=0x14197fac55d0 pc=0x38cf67f\n    fmt.(*pp).handleMethods(0x14197fc20000, 0x7fac58f0?)\n    \t/usr/lib/golang/src/fmt/print.go:674 +0x2f4 fp=0x14197fac58a8 sp=0x14197fac56e8 pc=0x1fdd1d4\n    fmt.(*pp).printArg(0x14197fc20000, {0x549a3a0, 0x14197fd92e80}, 0x76)\n    \t/usr/lib/golang/src/fmt/print.go:757 +0x42e fp=0x14197fac5938 sp=0x14197fac58a8 pc=0x1fdd9ee\n    fmt.(*pp).doPrintf(0x14197fc20000, {0x553a400, 0x2}, {0x14197fac5d08, 0x1, 0x1})\n    \t/usr/lib/golang/src/fmt/print.go:1075 +0x3ec fp=0x14197fac5a20 sp=0x14197fac5938 pc=0x1fe04ec\n    fmt.Sprintf({0x553a400, 0x2}, {0x14197fac5d08, 0x1, 0x1})\n    \t/usr/lib/golang/src/fmt/print.go:239 +0x53 fp=0x14197fac5a78 sp=0x14197fac5a20 pc=0x1fda6b3\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197fc4b200}, 0x14197f1c02a0)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:160 +0x8d8 fp=0x14197fac5d60 sp=0x14197fac5a78 pc=0x391bad8\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x14197fdd8140?}, {0x5a00300?, 0x14197fc4b200?}, 0x0?)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x14197fac5e88 sp=0x14197fac5d60 pc=0x3929c16\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14197fac5f78 sp=0x14197fac5e88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197fac5fe0 sp=0x14197fac5f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fac5fe8 sp=0x14197fac5fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 184 gp=0x14197fd3da40 m=nil [chan receive]:\n    runtime.gopark(0x14197f3c29a0?, 0x2034990?, 0x10?, 0xea?, 0x59f0ad0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14198011f9d8 sp=0x14198011f9b8 pc=0x1f38d2e\n    runtime.chanrecv(0x14197f3fa700, 0x14198011fcb8, 0x1)\n    \t/usr/lib/golang/src/runtime/chan.go:667 +0x4ae fp=0x14198011fa50 sp=0x14198011f9d8 pc=0x1ec7d2e\n    runtime.chanrecv2(0x5a09ed0?, 0x14197f530f90?)\n    \t/usr/lib/golang/src/runtime/chan.go:514 +0x12 fp=0x14198011fa78 sp=0x14198011fa50 pc=0x1ec7872\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197fc4b340}, 0x14197f1c02a0)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:141 +0x94d fp=0x14198011fd60 sp=0x14198011fa78 pc=0x391bb4d\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x74666968736e6570?, {0x59f0b40?, 0x14197fdd8140?}, {0x5a00300?, 0x14197fc4b340?}, 0x6d736f63222c2273?)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x14198011fe88 sp=0x14198011fd60 pc=0x3929c16\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14198011ff78 sp=0x14198011fe88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14198011ffe0 sp=0x14198011ff78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14198011ffe8 sp=0x14198011ffe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 135 gp=0x14197fb465a0 m=nil [semacquire]:\n    runtime.gopark(0x14197f454000?, 0x22dc6c0?, 0xa0?, 0x82?, 0x1419808322a0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fe99ce8 sp=0x14197fe99cc8 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fe99d50 sp=0x14197fe99ce8 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x14197fe99df0 sp=0x14197fe99d50 pc=0x1edbb12\n    runtime.gcAssistAlloc(0x14197fb465a0)\n    \t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x14197fe99e80 sp=0x14197fe99df0 pc=0x1ee06b6\n    runtime.deductAssistCredit(0x0?)\n    \t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x14197fe99ea0 sp=0x14197fe99e80 pc=0x1ed10a6\n    runtime.mallocgc(0x70, 0x0, 0x1)\n    \t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x14197fe99ed0 sp=0x14197fe99ea0 pc=0x1f35e07\n    runtime.makechan(0x0?, 0x0)\n    \t/usr/lib/golang/src/runtime/chan.go:99 +0x126 fp=0x14197fe99f10 sp=0x14197fe99ed0 pc=0x1ec6826\n    net/http.(*http2clientStream).cleanupWriteRequest(0x14197f2f1200, {0x0, 0x0})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8951 +0xd6 fp=0x14197fe99f90 sp=0x14197fe99f10 pc=0x22db1b6\n    net/http.(*http2clientStream).doRequest(0x14197f2f1200, 0x14197f7480a0?, 0x14197f4d0020?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8719 +0x28 fp=0x14197fe99fb8 sp=0x14197fe99f90 pc=0x22d99e8\n    net/http.(*http2ClientConn).roundTrip.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197fe99fe0 sp=0x14197fe99fb8 pc=0x22d999f\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fe99fe8 sp=0x14197fe99fe0 pc=0x1f40801\n    created by net/http.(*http2ClientConn).roundTrip in goroutine 110\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n    \n    goroutine 90 gp=0x14197fd82b40 m=nil [select]:\n    runtime.gopark(0x14197ff1df08?, 0x6?, 0x40?, 0x2b?, 0x14197ff1dda4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f203c08 sp=0x14197f203be8 pc=0x1f38d2e\n    runtime.selectgo(0x14197f203f08, 0x14197ff1dd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f203d38 sp=0x14197f203c08 pc=0x1f152a5\n    net/http.(*http2clientStream).writeRequest(0x14197f2f0f00, 0x14197fb50140, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x14197f203f90 sp=0x14197f203d38 pc=0x22da6a8\n    net/http.(*http2clientStream).doRequest(0x14197f2f0f00, 0x0?, 0x14197f1f1f98?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x14197f203fb8 sp=0x14197f203f90 pc=0x22d99d8\n    net/http.(*http2ClientConn).roundTrip.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197f203fe0 sp=0x14197f203fb8 pc=0x22d999f\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f203fe8 sp=0x14197f203fe0 pc=0x1f40801\n    created by net/http.(*http2ClientConn).roundTrip in goroutine 183\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n    \n    goroutine 225 gp=0x14197fd834a0 m=nil [sync.Cond.Wait]:\n    runtime.gopark(0x22b8a73?, 0x141980390000?, 0x0?, 0x60?, 0xe?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419801298f8 sp=0x1419801298d8 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    sync.runtime_notifyListWait(0x14197f2f0f48, 0x3c)\n    \t/usr/lib/golang/src/runtime/sema.go:617 +0x1b3 fp=0x141980129948 sp=0x1419801298f8 pc=0x1f3a8f3\n    sync.(*Cond).Wait(0x14197fbb1640?)\n    \t/usr/lib/golang/src/sync/cond.go:71 +0x73 fp=0x141980129980 sp=0x141980129948 pc=0x1f4d673\n    net/http.(*http2pipe).Read(0x14197f2f0f30, {0x14197fd98000, 0x1000, 0x1000})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:3928 +0xdb fp=0x1419801299e8 sp=0x141980129980 pc=0x22c2c1b\n    net/http.http2transportResponseBody.Read({0x141980129a60?}, {0x14197fd98000?, 0x20a6c63?, 0x1f3ec12?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9852 +0x59 fp=0x141980129a70 sp=0x1419801299e8 pc=0x22dfdd9\n    bufio.(*Reader).fill(0x14197fc90180)\n    \t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x141980129aa8 sp=0x141980129a70 pc=0x20a6c63\n    bufio.(*Reader).ReadByte(0x14197fc90180)\n    \t/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x141980129ac0 sp=0x141980129aa8 pc=0x20a73c7\n    compress/flate.(*decompressor).huffSym(0x141980058608, 0x141980058638)\n    \t/usr/lib/golang/src/compress/flate/inflate.go:720 +0x7e fp=0x141980129b10 sp=0x141980129ac0 pc=0x228893e\n    compress/flate.(*decompressor).huffmanBlock(0x82bed80?)\n    \t/usr/lib/golang/src/compress/flate/inflate.go:495 +0x45 fp=0x141980129b90 sp=0x141980129b10 pc=0x2287605\n    compress/flate.(*decompressor).Read(0x141980058608, {0x14197fd99cea, 0x316, 0xcea?})\n    \t/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x141980129bc0 sp=0x141980129b90 pc=0x2286dbb\n    compress/gzip.(*Reader).Read(0x14197f3d8848, {0x14197fd99cea, 0x316, 0x316})\n    \t/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x141980129c50 sp=0x141980129bc0 pc=0x228c838\n    github.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x141980129cf0?, {0x14197fd99cea?, 0x1000?, 0x0?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x141980129c80 sp=0x141980129c50 pc=0x38b33e2\n    bufio.(*Reader).fill(0x14197f546110)\n    \t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x141980129cb8 sp=0x141980129c80 pc=0x20a6c63\n    bufio.(*Reader).ReadSlice(0x14197f546110, 0xa)\n    \t/usr/lib/golang/src/bufio/bufio.go:377 +0x29 fp=0x141980129d00 sp=0x141980129cb8 pc=0x20a7769\n    bufio.(*Reader).collectFragments(0x14197f546110, 0xa)\n    \t/usr/lib/golang/src/bufio/bufio.go:452 +0x85 fp=0x141980129de0 sp=0x141980129d00 pc=0x20a7b85\n    bufio.(*Reader).ReadBytes(0x4bbc5a0?, 0xa0?)\n    \t/usr/lib/golang/src/bufio/bufio.go:479 +0x17 fp=0x141980129e60 sp=0x141980129de0 pc=0x20a7e77\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.(*frameReader).advance(0x14197f546100)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:64 +0x45 fp=0x141980129eb0 sp=0x141980129e60 pc=0x38d6d45\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readRoutine(0x14197f546100, 0x14197f8c54f0)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:85 +0x51 fp=0x141980129fc0 sp=0x141980129eb0 pc=0x38d7591\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap2()\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x1b fp=0x141980129fe0 sp=0x141980129fc0 pc=0x38d74db\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x141980129fe8 sp=0x141980129fe0 pc=0x1f40801\n    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 183\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x24a\n    \n    goroutine 174 gp=0x14197ff64000 m=nil [select]:\n    runtime.gopark(0x1419800b9df8?, 0x2?, 0x0?, 0x0?, 0x1419800b9db4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf3c50 sp=0x14197fcf3c30 pc=0x1f38d2e\n    runtime.selectgo(0x14197fcf3df8, 0x1419800b9db0, 0x59d87c8?, 0x0, 0x14197f2980e0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197fcf3d80 sp=0x14197fcf3c50 pc=0x1f152a5\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.nextFrame(0x14197f8c8370)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:198 +0x96 fp=0x14197fcf3e38 sp=0x14197fcf3d80 pc=0x38d7d16\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readPrimaryTable(0x14197f8c8370, 0x14197fb50000)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:258 +0xb1 fp=0x14197fcf3ec8 sp=0x14197fcf3e38 pc=0x38d8171\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x14197f8c8370)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:174 +0x189 fp=0x14197fcf3f30 sp=0x14197fcf3ec8 pc=0x38d7b89\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x14197f8c8370, 0x14197f59aee0)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x14197fcf3fc0 sp=0x14197fcf3f30 pc=0x38d788c\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x14197fcf3fe0 sp=0x14197fcf3fc0 pc=0x38d751b\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf3fe8 sp=0x14197fcf3fe0 pc=0x1f40801\n    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 184\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc\n    \n    rax    0x0\n    rbx    0x7f808bfff640\n    rcx    0x7f80df60594c\n    rdx    0x6\n    rdi    0x109fd\n    rsi    0x10a00\n    rbp    0x10a00\n    rsp    0x7f808bffe790\n    r8     0x7f808bffe860\n    r9     0x7f80df72b4e0\n    r10    0x8\n    r11    0x246\n    r12    0x6\n    r13    0x7f808bffeb60\n    r14    0x14197f1334a0\n    r15    0xffffffffffffffff\n    rip    0x7f80df60594c\n    rflags 0x246\n    cs     0x33\n    fs     0x0\n    gs     0x0\n    \n      query-infra-svc: command failed: exit status 2\n    output: runtime/cgo: pthread_create failed: Resource temporarily unavailable\n    SIGABRT: abort\n    PC=0x7f7ef9baa94c m=3 sigcode=18446744073709551610\n    \n    goroutine 0 gp=0x170ac5c1f0e0 m=3 mp=0x170ac5c91808 [idle]:\n    runtime: g 0 gp=0x170ac5c1f0e0: unknown pc 0x7f7ef9baa94c\n    stack: frame={sp:0x7f7eabffe790, fp:0x0} stack=[0x7f7eab7ff1e0,0x7f7eabffede0)\n                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n    00007f7eabffe690: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6a0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6b0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6c0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6d0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6e0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6f0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe700: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe710: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe720: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe730: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe740: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe750: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe760: 00000000 00000000  00007f7e f9bb7fa9  ............~...\n    00007f7eabffe770: 00000000 00000000  00000000 00000011  ................\n    00007f7eabffe780: 00000000 00000130  00007f7e f9baa93e  0.......\u003e...~...\n    00007f7eabffe790: 00000000 00000120  00000000 00000000   ...............\n                      ^ SP\n    00007f7eabffe7a0: 00000000 00000013  00000000 00000000  ................\n    00007f7eabffe7b0: 00000000 00000000  00007f7e f9c2dcff  ............~...\n    00007f7eabffe7c0: 00007f7e 7bfff910  00007f7e f9c2dcb1  ...{~.......~...\n    00007f7eabffe7d0: 00007f7e 7bfff640  00007f7e abffeaa0  @..{~.......~...\n    00007f7eabffe7e0: 00007f7e abffe90e  00007f7e abffe90f  ....~.......~...\n    00007f7eabffe7f0: 00007f7e 7bfff640  00007f7e f9ba87f5  @..{~.......~...\n    00007f7eabffe800: 00007f7e a4000020  00000000 00000120   ...~... .......\n    00007f7eabffe810: 00000000 003d0f00  176a34c4 bac87500  ..=......u...4j.\n    00007f7eabffe820: 00007f7e abfff640  00000000 00000006  @...~...........\n    00007f7eabffe830: 00007f7e a4000f30  00007f7e abffeb60  0...~...`...~...\n    00007f7eabffe840: 0000170a c5c1f0e0  00007f7e f9b5d646  ........F...~...\n    00007f7eabffe850: 00007f7e f9d1ae90  00007f7e f9b477f3  ....~....w..~...\n    00007f7eabffe860: 00000000 00000020  176a34c4 bac87500   ........u...4j.\n    00007f7eabffe870: 00007f7e 7b7ff000  00007f7e f9c1cac4  ...{~.......~...\n    00007f7eabffe880: 00000000 00000000  00000000 0000000d  ................\n    runtime: g 0 gp=0x170ac5c1f0e0: unknown pc 0x7f7ef9baa94c\n    stack: frame={sp:0x7f7eabffe790, fp:0x0} stack=[0x7f7eab7ff1e0,0x7f7eabffede0)\n                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n    00007f7eabffe690: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6a0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6b0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6c0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6d0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6e0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6f0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe700: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe710: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe720: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe730: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe740: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe750: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe760: 00000000 00000000  00007f7e f9bb7fa9  ............~...\n    00007f7eabffe770: 00000000 00000000  00000000 00000011  ................\n    00007f7eabffe780: 00000000 00000130  00007f7e f9baa93e  0.......\u003e...~...\n    00007f7eabffe790: 00000000 00000120  00000000 00000000   ...............\n                      ^ SP\n    00007f7eabffe7a0: 00000000 00000013  00000000 00000000  ................\n    00007f7eabffe7b0: 00000000 00000000  00007f7e f9c2dcff  ............~...\n    00007f7eabffe7c0: 00007f7e 7bfff910  00007f7e f9c2dcb1  ...{~.......~...\n    00007f7eabffe7d0: 00007f7e 7bfff640  00007f7e abffeaa0  @..{~.......~...\n    00007f7eabffe7e0: 00007f7e abffe90e  00007f7e abffe90f  ....~.......~...\n    00007f7eabffe7f0: 00007f7e 7bfff640  00007f7e f9ba87f5  @..{~.......~...\n    00007f7eabffe800: 00007f7e a4000020  00000000 00000120   ...~... .......\n    00007f7eabffe810: 00000000 003d0f00  176a34c4 bac87500  ..=......u...4j.\n    00007f7eabffe820: 00007f7e abfff640  00000000 00000006  @...~...........\n    00007f7eabffe830: 00007f7e a4000f30  00007f7e abffeb60  0...~...`...~...\n    00007f7eabffe840: 0000170a c5c1f0e0  00007f7e f9b5d646  ........F...~...\n    00007f7eabffe850: 00007f7e f9d1ae90  00007f7e f9b477f3  ....~....w..~...\n    00007f7eabffe860: 00000000 00000020  176a34c4 bac87500   ........u...4j.\n    00007f7eabffe870: 00007f7e 7b7ff000  00007f7e f9c1cac4  ...{~.......~...\n    00007f7eabffe880: 00000000 00000000  00000000 0000000d  ................\n    \n    goroutine 1 gp=0x170ac5c1e1e0 m=nil [sync.WaitGroup.Wait]:\n    runtime.gopark(0x170ac5cb27b8?, 0x2022a4d?, 0x70?, 0x8e?, 0x170ac5cb27e8?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cb27a0 sp=0x170ac5cb2780 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x170ac65f8210, 0x0, 0x1, 0x0, 0x19)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x170ac5cb2808 sp=0x170ac5cb27a0 pc=0x1f162b2\n    sync.runtime_SemacquireWaitGroup(0x170ac6709950?, 0xe0?)\n    \t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x170ac5cb2840 sp=0x170ac5cb2808 pc=0x1f3a66e\n    sync.(*WaitGroup).Wait(0x170ac65f8208)\n    \t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x170ac5cb2868 sp=0x170ac5cb2840 pc=0x1f4f445\n    golang.org/x/sync/errgroup.(*Group).Wait(0x170ac65f8200)\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x170ac5cb2888 sp=0x170ac5cb2868 pc=0x362507e\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0}, {0x170ac626a1e0, 0x6, 0x6})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x170ac5cb2948 sp=0x170ac5cb2888 pc=0x3927e05\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).gatherInfraLogs(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:595 +0xadd fp=0x170ac5cb2d30 sp=0x170ac5cb2948 pc=0x392799d\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:333 +0x533 fp=0x170ac5cb3908 sp=0x170ac5cb2d30 pc=0x3921bf3\n    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedInfraQueryOptions).RunInfra(0x170ac5c2c108, {0x59f0b08, 0x170ac64108d0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:83 +0x71f fp=0x170ac5cb3b30 sp=0x170ac5cb3908 pc=0x393887f\n    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawInfraQueryOptions).Run(0x0?, {0x59f0b08, 0x170ac64108d0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_options.go:109 +0x54 fp=0x170ac5cb3b58 sp=0x170ac5cb3b30 pc=0x3939134\n    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryInfraCommand.func1(0x170ac66f5508?, {0x170ac5d40960?, 0x4?, 0x553b0e2?})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:45 +0x28 fp=0x170ac5cb3b80 sp=0x170ac5cb3b58 pc=0x3938128\n    github.com/spf13/cobra.(*Command).execute(0x170ac66f5508, {0x170ac5e36100, 0xa, 0xa})\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x170ac5cb3d28 sp=0x170ac5cb3b80 pc=0x212e614\n    github.com/spf13/cobra.(*Command).ExecuteC(0x170ac6228608)\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x170ac5cb3e18 sp=0x170ac5cb3d28 pc=0x212efa5\n    github.com/spf13/cobra.(*Command).Execute(...)\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071\n    github.com/spf13/cobra.(*Command).ExecuteContext(...)\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064\n    main.main()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x170ac5cb3f48 sp=0x170ac5cb3e18 pc=0x47ddd6a\n    runtime.main()\n    \t/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x170ac5cb3fe0 sp=0x170ac5cb3f48 pc=0x1f01975\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cb3fe8 sp=0x170ac5cb3fe0 pc=0x1f40801\n    \n    goroutine 2 gp=0x170ac5c1ed20 m=nil [force gc (idle)]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdefa8 sp=0x170ac5cdef88 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.forcegchelper()\n    \t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x170ac5cdefe0 sp=0x170ac5cdefa8 pc=0x1f01c93\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdefe8 sp=0x170ac5cdefe0 pc=0x1f40801\n    created by runtime.init.7 in goroutine 1\n    \t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\n    \n    goroutine 18 gp=0x170ac5d843c0 m=nil [GC sweep wait]:\n    runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cda788 sp=0x170ac5cda768 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.bgsweep(0x170ac5d92000)\n    \t/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x170ac5cda7c8 sp=0x170ac5cda788 pc=0x1ee9f11\n    runtime.gcenable.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x170ac5cda7e0 sp=0x170ac5cda7c8 pc=0x1edadd7\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cda7e8 sp=0x170ac5cda7e0 pc=0x1f40801\n    created by runtime.gcenable in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\n    \n    goroutine 19 gp=0x170ac5d845a0 m=nil [GC scavenge wait]:\n    runtime.gopark(0x10000?, 0x5981cd8?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdaf78 sp=0x170ac5cdaf58 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.(*scavengerState).park(0x82979c0)\n    \t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x170ac5cdafa8 sp=0x170ac5cdaf78 pc=0x1ee77e9\n    runtime.bgscavenge(0x170ac5d92000)\n    \t/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x170ac5cdafc8 sp=0x170ac5cdafa8 pc=0x1ee7d99\n    runtime.gcenable.gowrap2()\n    \t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x170ac5cdafe0 sp=0x170ac5cdafc8 pc=0x1edad97\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdafe8 sp=0x170ac5cdafe0 pc=0x1f40801\n    created by runtime.gcenable in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\n    \n    goroutine 34 gp=0x170ac5e00000 m=nil [GOMAXPROCS updater (idle)]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e06788 sp=0x170ac5e06768 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.updateMaxProcsGoroutine()\n    \t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x170ac5e067e0 sp=0x170ac5e06788 pc=0x1f10267\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e067e8 sp=0x170ac5e067e0 pc=0x1f40801\n    created by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\n    \t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\n    \n    goroutine 35 gp=0x170ac5e005a0 m=nil [finalizer wait]:\n    runtime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x170ac5cde601?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cde620 sp=0x170ac5cde600 pc=0x1f38d2e\n    runtime.runFinalizers()\n    \t/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x170ac5cde7e0 sp=0x170ac5cde620 pc=0x1ed9cc7\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cde7e8 sp=0x170ac5cde7e0 pc=0x1f40801\n    created by runtime.createfing in goroutine 1\n    \t/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d\n    \n    goroutine 36 gp=0x170ac5ff81e0 m=nil [cleanup wait]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e06f68 sp=0x170ac5e06f48 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.(*cleanupQueue).dequeue(0x82980e0)\n    \t/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x170ac5e06fa0 sp=0x170ac5e06f68 pc=0x1ed6074\n    runtime.runCleanups()\n    \t/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x170ac5e06fe0 sp=0x170ac5e06fa0 pc=0x1ed66e5\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e06fe8 sp=0x170ac5e06fe0 pc=0x1f40801\n    created by runtime.(*cleanupQueue).createGs in goroutine 1\n    \t/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5\n    \n    goroutine 37 gp=0x170ac611bc20 m=nil [GC worker (idle)]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e07740 sp=0x170ac5e07720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x170ac5e0b9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5e077c8 sp=0x170ac5e07740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5e077e0 sp=0x170ac5e077c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e077e8 sp=0x170ac5e077e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 3 gp=0x170ac5c1f680 m=nil [runnable]:\n    runtime.gopark(0x125c26559f5b?, 0x3?, 0x94?, 0x85?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdf740 sp=0x170ac5cdf720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x170ac5e0b9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdf7c8 sp=0x170ac5cdf740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdf7e0 sp=0x170ac5cdf7c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdf7e8 sp=0x170ac5cdf7e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 20 gp=0x170ac5d84780 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125c265c3db1?, 0x3?, 0x92?, 0xcd?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdb740 sp=0x170ac5cdb720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x170ac5e0b9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdb7c8 sp=0x170ac5cdb740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdb7e0 sp=0x170ac5cdb7c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdb7e8 sp=0x170ac5cdb7e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 38 gp=0x170ac6134000 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125c26588b9e?, 0x1?, 0xf7?, 0xb4?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e07f40 sp=0x170ac5e07f20 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x170ac5e0b9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5e07fc8 sp=0x170ac5e07f40 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5e07fe0 sp=0x170ac5e07fc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e07fe8 sp=0x170ac5e07fe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 4 gp=0x170ac5c1f860 m=10 mp=0x170ac6280008 [GC mark termination]:\n    runtime.systemstack_switch()\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x170ac5cdfea0 sp=0x170ac5cdfe90 pc=0x1f3ebc8\n    runtime.forEachP(...)\n    \t/usr/lib/golang/src/runtime/proc.go:2112\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x170ac5cdff40 sp=0x170ac5cdfea0 pc=0x1edbc34\n    runtime.gcBgMarkWorker(0x170ac5e0b9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x170ac5cdffc8 sp=0x170ac5cdff40 pc=0x1edda88\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdffe0 sp=0x170ac5cdffc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdffe8 sp=0x170ac5cdffe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 21 gp=0x170ac5d84960 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125c265818cf?, 0x3?, 0xa5?, 0x9c?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdbf40 sp=0x170ac5cdbf20 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x170ac5e0b9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdbfc8 sp=0x170ac5cdbf40 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdbfe0 sp=0x170ac5cdbfc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdbfe8 sp=0x170ac5cdbfe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 22 gp=0x170ac5d84b40 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125c265846e7?, 0x3?, 0x77?, 0xb6?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdc740 sp=0x170ac5cdc720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x170ac5e0b9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdc7c8 sp=0x170ac5cdc740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdc7e0 sp=0x170ac5cdc7c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdc7e8 sp=0x170ac5cdc7e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 23 gp=0x170ac5d84d20 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125c26581951?, 0x3?, 0x91?, 0x58?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdcf40 sp=0x170ac5cdcf20 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x170ac5e0b9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdcfc8 sp=0x170ac5cdcf40 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdcfe0 sp=0x170ac5cdcfc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdcfe8 sp=0x170ac5cdcfe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 39 gp=0x170ac648a3c0 m=nil [select, locked to thread]:\n    runtime.gopark(0x170ac5e097a8?, 0x2?, 0x1?, 0xa3?, 0x170ac5e09794?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e09628 sp=0x170ac5e09608 pc=0x1f38d2e\n    runtime.selectgo(0x170ac5e097a8, 0x170ac5e09790, 0x0?, 0x0, 0x0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5e09758 sp=0x170ac5e09628 pc=0x1f152a5\n    runtime.ensureSigM.func1()\n    \t/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x170ac5e097e0 sp=0x170ac5e09758 pc=0x1f328e8\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e097e8 sp=0x170ac5e097e0 pc=0x1f40801\n    created by runtime.ensureSigM in goroutine 1\n    \t/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5\n    \n    goroutine 24 gp=0x170ac62f4960 m=8 mp=0x170ac6180008 [syscall]:\n    runtime.notetsleepg(0x82bdea0, 0xffffffffffffffff)\n    \t/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x170ac5e037a0 sp=0x170ac5e03778 pc=0x1ece509\n    os/signal.signal_recv()\n    \t/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x170ac5e037c0 sp=0x170ac5e037a0 pc=0x1f3b298\n    os/signal.loop()\n    \t/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x170ac5e037e0 sp=0x170ac5e037c0 pc=0x2030553\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e037e8 sp=0x170ac5e037e0 pc=0x1f40801\n    created by os/signal.Notify.func1.1 in goroutine 1\n    \t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\n    \n    goroutine 50 gp=0x170ac6135a40 m=nil [select]:\n    runtime.gopark(0x170ac66e3798?, 0x2?, 0x80?, 0x6f?, 0x170ac66e378c?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac66e3628 sp=0x170ac66e3608 pc=0x1f38d2e\n    runtime.selectgo(0x170ac66e3798, 0x170ac66e3788, 0x0?, 0x0, 0x0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac66e3758 sp=0x170ac66e3628 pc=0x1f152a5\n    os/signal.NotifyContext.func1()\n    \t/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x170ac66e37e0 sp=0x170ac66e3758 pc=0x202ff9c\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac66e37e8 sp=0x170ac66e37e0 pc=0x1f40801\n    created by os/signal.NotifyContext in goroutine 1\n    \t/usr/lib/golang/src/os/signal/signal.go:291 +0x158\n    \n    goroutine 102 gp=0x170ac61843c0 m=nil [select]:\n    runtime.gopark(0x170ac5cefc70?, 0x2?, 0xc8?, 0xf9?, 0x170ac5cefb6c?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cef9f0 sp=0x170ac5cef9d0 pc=0x1f38d2e\n    runtime.selectgo(0x170ac5cefc70, 0x170ac5cefb68, 0x554857c?, 0x0, 0x226?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5cefb20 sp=0x170ac5cef9f0 pc=0x1f152a5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x170ac5c49130}, 0x170ac65fa070, 0x170ac6410cf0)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:283 +0x21b fp=0x170ac5ceff48 sp=0x170ac5cefb20 pc=0x39204fb\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x170ac5ceff78 sp=0x170ac5ceff48 pc=0x392800c\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5ceffe0 sp=0x170ac5ceff78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5ceffe8 sp=0x170ac5ceffe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 104 gp=0x170ac6184d20 m=nil [select]:\n    runtime.gopark(0x170ac6992ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69929d8?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6992848 sp=0x170ac6992828 pc=0x1f38d2e\n    runtime.selectgo(0x170ac6992ac8, 0x170ac69929d0, 0x55562e1?, 0x0, 0x1f6d6cf?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6992978 sp=0x170ac6992848 pc=0x1f152a5\n    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac619fb80, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac6992b40 sp=0x170ac6992978 pc=0x22d92e5\n    net/http.(*http2ClientConn).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8603\n    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac619fb80, {0x4c?, 0xfa?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac6992c80 sp=0x170ac6992b40 pc=0x22d50ad\n    net/http.(*http2Transport).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7852\n    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac619fb80?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac6992cc0 sp=0x170ac6992c80 pc=0x22e3a9c\n    net/http.(*Transport).roundTrip(0x8175400, 0x170ac619fa40)\n    \t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac6992ea0 sp=0x170ac6992cc0 pc=0x231037f\n    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n    \t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac6992ec0 sp=0x170ac6992ea0 pc=0x2321178\n    net/http.send(0x170ac619fa40, {0x59a1ce0, 0x8175400}, {0x170ac6993138?, 0x1f3bac6?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69930b0 sp=0x170ac6992ec0 pc=0x22ad9ab\n    net/http.(*Client).send(0x170ac6410c60, 0x170ac619fa40, {0x170ac69931e8?, 0x1fdd787?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac6993148 sp=0x170ac69930b0 pc=0x22ad1d8\n    net/http.(*Client).do(0x170ac6410c60, 0x170ac619fa40)\n    \t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac6993338 sp=0x170ac6993148 pc=0x22af817\n    net/http.(*Client).Do(...)\n    \t/usr/lib/golang/src/net/http/client.go:592\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5c49450}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e34a0}, 0x170ac65f7ce0, {0x170ac65d9560, 0x112})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac6993630 sp=0x170ac6993338 pc=0x38de6bd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5c49450}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x170ac65f7c80, 0x0, {0x0, ...}, ...})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac6993888 sp=0x170ac6993630 pc=0x38ddecd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5c49450?}, 0xa8?, {0x170ac5db1a50?, 0x170ac6764a48?}, 0x3?, 0x2022a4d?)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac6993960 sp=0x170ac6993888 pc=0x38dd40b\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194700?}, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x170ac6764a48, 0x3, 0x4})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69939e0 sp=0x170ac6993960 pc=0x38e51e2\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194700}, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x0?, 0x2?, 0x1?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac6993a78 sp=0x170ac69939e0 pc=0x38e4eaf\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020500}, 0x170ac65fa000)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac6993d60 sp=0x170ac6993a78 pc=0x391b5c5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac5fe1880?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020500?}, 0x0?)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac6993e88 sp=0x170ac6993d60 pc=0x3929c16\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac6993f78 sp=0x170ac6993e88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac6993fe0 sp=0x170ac6993f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6993fe8 sp=0x170ac6993fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 101 gp=0x170ac61d4000 m=nil [sync.WaitGroup.Wait]:\n    runtime.gopark(0x170ac5de7e00?, 0x1ed0765?, 0xb0?, 0xfa?, 0xe?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de7da0 sp=0x170ac5de7d80 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x170ac65f8290, 0x0, 0x1, 0x0, 0x19)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x170ac5de7e08 sp=0x170ac5de7da0 pc=0x1f162b2\n    sync.runtime_SemacquireWaitGroup(0x170ac6709a70?, 0x0?)\n    \t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x170ac5de7e40 sp=0x170ac5de7e08 pc=0x1f3a66e\n    sync.(*WaitGroup).Wait(0x170ac65f8288)\n    \t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x170ac5de7e68 sp=0x170ac5de7e40 pc=0x1f4f445\n    golang.org/x/sync/errgroup.(*Group).Wait(0x170ac65f8280)\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x170ac5de7e88 sp=0x170ac5de7e68 pc=0x362507e\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x170ac5fe1140, {0x59f0b40, 0x170ac5c49130}, {0x170ac626a1e0, 0x6, 0x170ac5cac1c0?}, 0x170ac65fa000)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x170ac5de7f00 sp=0x170ac5de7e88 pc=0x392976a\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x170ac5de7f78 sp=0x170ac5de7f00 pc=0x392808e\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5de7fe0 sp=0x170ac5de7f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de7fe8 sp=0x170ac5de7fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 67 gp=0x170ac62f5a40 m=nil [IO wait]:\n    runtime.gopark(0x170ac6497780?, 0x8?, 0x40?, 0x16?, 0xb?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de47b8 sp=0x170ac5de4798 pc=0x1f38d2e\n    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n    \t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x170ac5de47f0 sp=0x170ac5de47b8 pc=0x1ef9e57\n    internal/poll.runtime_pollWait(0x7f7eb1670000, 0x72)\n    \t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x170ac5de4810 sp=0x170ac5de47f0 pc=0x1f375c5\n    internal/poll.(*pollDesc).wait(0x170ac620b480?, 0x170ac65a2000?, 0x0)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x170ac5de4838 sp=0x170ac5de4810 pc=0x1fbd167\n    internal/poll.(*pollDesc).waitRead(...)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\n    internal/poll.(*FD).Read(0x170ac620b480, {0x170ac65a2000, 0x3500, 0x3500})\n    \t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x170ac5de48d0 sp=0x170ac5de4838 pc=0x1fbe4ce\n    net.(*netFD).Read(0x170ac620b480, {0x170ac65a2000?, 0x170ac5de4901?, 0x22a2e75?})\n    \t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x170ac5de4918 sp=0x170ac5de48d0 pc=0x20798c5\n    net.(*conn).Read(0x170ac62823d8, {0x170ac65a2000?, 0x7f7eb047fc00?, 0x7f7ef9af2e00?})\n    \t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x170ac5de4960 sp=0x170ac5de4918 pc=0x208b7a5\n    crypto/tls.(*atLeastReader).Read(0x170ac6708f30, {0x170ac65a2000?, 0x170ac5de4a48?, 0x1efcc28?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x170ac5de49a8 sp=0x170ac5de4960 pc=0x2222ddb\n    bytes.(*Buffer).ReadFrom(0x170ac605fb28, {0x59a3680, 0x170ac6708f30})\n    \t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x170ac5de4a00 sp=0x170ac5de49a8 pc=0x1fe9258\n    crypto/tls.(*Conn).readFromUntil(0x170ac605f888, {0x59a32c0, 0x170ac62823d8}, 0x222764c?)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x170ac5de4a38 sp=0x170ac5de4a00 pc=0x2222fbe\n    crypto/tls.(*Conn).readRecordOrCCS(0x170ac605f888, 0x0)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x170ac5de4c18 sp=0x170ac5de4a38 pc=0x222009b\n    crypto/tls.(*Conn).readRecord(...)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:588\n    crypto/tls.(*Conn).Read(0x170ac605f888, {0x170ac6029000, 0x1000, 0x22c34e0?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x170ac5de4c80 sp=0x170ac5de4c18 pc=0x22267c5\n    bufio.(*Reader).Read(0x170ac647aa20, {0x170ac6153624, 0x9, 0x22e100e?})\n    \t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0x170ac5de4cb8 sp=0x170ac5de4c80 pc=0x20a71f7\n    io.ReadAtLeast({0x59a1260, 0x170ac647aa20}, {0x170ac6153624, 0x9, 0x9}, 0x9)\n    \t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0x170ac5de4d00 sp=0x170ac5de4cb8 pc=0x1fb17ee\n    io.ReadFull(...)\n    \t/usr/lib/golang/src/io/io.go:354\n    net/http.http2readFrameHeader({0x170ac6153624, 0x9, 0x170a00000005?}, {0x59a1260?, 0x170ac647aa20?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:1805 +0x65 fp=0x170ac5de4d50 sp=0x170ac5de4d00 pc=0x22b9e25\n    net/http.(*http2Framer).ReadFrameHeader(0x170ac61535e0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:2071 +0x6b fp=0x170ac5de4dd8 sp=0x170ac5de4d50 pc=0x22ba9eb\n    net/http.(*http2Framer).ReadFrame(0x170ac61535e0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:2130 +0x18 fp=0x170ac5de4e18 sp=0x170ac5de4dd8 pc=0x22bb1b8\n    net/http.(*http2clientConnReadLoop).run(0x170ac5de4fa8)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9550 +0xca fp=0x170ac5de4f60 sp=0x170ac5de4e18 pc=0x22de14a\n    net/http.(*http2ClientConn).readLoop(0x170ac61d9880)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x170ac5de4fc8 sp=0x170ac5de4f60 pc=0x22dd532\n    net/http.(*http2Transport).newClientConn.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x170ac5de4fe0 sp=0x170ac5de4fc8 pc=0x22d7057\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de4fe8 sp=0x170ac5de4fe0 pc=0x1f40801\n    created by net/http.(*http2Transport).newClientConn in goroutine 66\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5\n    \n    goroutine 73 gp=0x170ac648ad20 m=nil [IO wait]:\n    runtime.gopark(0x170ac6497e40?, 0x1edb176?, 0x80?, 0xed?, 0xb?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de6808 sp=0x170ac5de67e8 pc=0x1f38d2e\n    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n    \t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x170ac5de6840 sp=0x170ac5de6808 pc=0x1ef9e57\n    internal/poll.runtime_pollWait(0x7f7eb166fe00, 0x72)\n    \t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x170ac5de6860 sp=0x170ac5de6840 pc=0x1f375c5\n    internal/poll.(*pollDesc).wait(0x170ac60c4e80?, 0x170ac5ecd300?, 0x0)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x170ac5de6888 sp=0x170ac5de6860 pc=0x1fbd167\n    internal/poll.(*pollDesc).waitRead(...)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\n    internal/poll.(*FD).Read(0x170ac60c4e80, {0x170ac5ecd300, 0x1300, 0x1300})\n    \t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x170ac5de6920 sp=0x170ac5de6888 pc=0x1fbe4ce\n    net.(*netFD).Read(0x170ac60c4e80, {0x170ac5ecd300?, 0x170ac5ecd30a?, 0x5?})\n    \t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x170ac5de6968 sp=0x170ac5de6920 pc=0x20798c5\n    net.(*conn).Read(0x170ac5d82278, {0x170ac5ecd300?, 0x7f7eb047fc00?, 0x7f7ef9af2e00?})\n    \t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x170ac5de69b0 sp=0x170ac5de6968 pc=0x208b7a5\n    crypto/tls.(*atLeastReader).Read(0x170ac67088e8, {0x170ac5ecd300?, 0x170ac5de6a98?, 0x1efcc28?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x170ac5de69f8 sp=0x170ac5de69b0 pc=0x2222ddb\n    bytes.(*Buffer).ReadFrom(0x170ac5dd02a8, {0x59a3680, 0x170ac67088e8})\n    \t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x170ac5de6a50 sp=0x170ac5de69f8 pc=0x1fe9258\n    crypto/tls.(*Conn).readFromUntil(0x170ac5dd0008, {0x59a32c0, 0x170ac5d82278}, 0x222764c?)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x170ac5de6a88 sp=0x170ac5de6a50 pc=0x2222fbe\n    crypto/tls.(*Conn).readRecordOrCCS(0x170ac5dd0008, 0x0)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x170ac5de6c68 sp=0x170ac5de6a88 pc=0x222009b\n    crypto/tls.(*Conn).readRecord(...)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:588\n    crypto/tls.(*Conn).Read(0x170ac5dd0008, {0x170ac6564000, 0x1000, 0x4e55540?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x170ac5de6cd0 sp=0x170ac5de6c68 pc=0x22267c5\n    net/http.(*persistConn).Read(0x170ac5f73180, {0x170ac6564000?, 0x59a0ea0?, 0x81260e0?})\n    \t/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x170ac5de6d30 sp=0x170ac5de6cd0 pc=0x2319d07\n    bufio.(*Reader).fill(0x170ac65537a0)\n    \t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x170ac5de6d68 sp=0x170ac5de6d30 pc=0x20a6c63\n    bufio.(*Reader).Peek(0x170ac65537a0, 0x1)\n    \t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x170ac5de6d80 sp=0x170ac5de6d68 pc=0x20a6df3\n    net/http.(*persistConn).readLoop(0x170ac5f73180)\n    \t/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x170ac5de6fc8 sp=0x170ac5de6d80 pc=0x231a812\n    net/http.(*Transport).dialConn.gowrap2()\n    \t/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x170ac5de6fe0 sp=0x170ac5de6fc8 pc=0x2319197\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de6fe8 sp=0x170ac5de6fe0 pc=0x1f40801\n    created by net/http.(*Transport).dialConn in goroutine 82\n    \t/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac\n    \n    goroutine 103 gp=0x170ac648af00 m=nil [select]:\n    runtime.gopark(0x170ac5de0d00?, 0x2?, 0xa0?, 0x6a?, 0x170ac5de0c74?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de0ac8 sp=0x170ac5de0aa8 pc=0x1f38d2e\n    runtime.selectgo(0x170ac5de0d00, 0x170ac5de0c70, 0x553bfdc?, 0x0, 0x5a48c80?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5de0bf8 sp=0x170ac5de0ac8 pc=0x1f152a5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x0?, {0x59f0b40, 0x170ac5c49130}, 0x170ac65fa000, 0x170ac65fa070)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x170ac5de0f10 sp=0x170ac5de0bf8 pc=0x392836e\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x170ac5de0f78 sp=0x170ac5de0f10 pc=0x3927f3d\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5de0fe0 sp=0x170ac5de0f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de0fe8 sp=0x170ac5de0fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 74 gp=0x170ac648b4a0 m=nil [select]:\n    runtime.gopark(0x170ac5df1f48?, 0x2?, 0x0?, 0xb6?, 0x170ac5df1ef4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6760d88 sp=0x170ac6760d68 pc=0x1f38d2e\n    runtime.selectgo(0x170ac6760f48, 0x170ac5df1ef0, 0x170ac68d6980?, 0x0, 0x170ac655e8d0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6760eb8 sp=0x170ac6760d88 pc=0x1f152a5\n    net/http.(*persistConn).writeLoop(0x170ac5f73180)\n    \t/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x170ac6760fc8 sp=0x170ac6760eb8 pc=0x231c426\n    net/http.(*Transport).dialConn.gowrap3()\n    \t/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x170ac6760fe0 sp=0x170ac6760fc8 pc=0x2319157\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6760fe8 sp=0x170ac6760fe0 pc=0x1f40801\n    created by net/http.(*Transport).dialConn in goroutine 82\n    \t/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe\n    \n    goroutine 105 gp=0x170ac61d43c0 m=nil [select]:\n    runtime.gopark(0x170ac699cac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac699c9d8?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac699c848 sp=0x170ac699c828 pc=0x1f38d2e\n    runtime.selectgo(0x170ac699cac8, 0x170ac699c9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac699c978 sp=0x170ac699c848 pc=0x1f152a5\n    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e000, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac699cb40 sp=0x170ac699c978 pc=0x22d92e5\n    net/http.(*http2ClientConn).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8603\n    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e000, {0x4c?, 0xfa?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac699cc80 sp=0x170ac699cb40 pc=0x22d50ad\n    net/http.(*http2Transport).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7852\n    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e000?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac699ccc0 sp=0x170ac699cc80 pc=0x22e3a9c\n    net/http.(*Transport).roundTrip(0x8175400, 0x170ac619fcc0)\n    \t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac699cea0 sp=0x170ac699ccc0 pc=0x231037f\n    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n    \t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac699cec0 sp=0x170ac699cea0 pc=0x2321178\n    net/http.send(0x170ac619fcc0, {0x59a1ce0, 0x8175400}, {0x170ac699d138?, 0x1f3bac6?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac699d0b0 sp=0x170ac699cec0 pc=0x22ad9ab\n    net/http.(*Client).send(0x170ac6410c60, 0x170ac619fcc0, {0x170ac699d1e8?, 0x1fdd787?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac699d148 sp=0x170ac699d0b0 pc=0x22ad1d8\n    net/http.(*Client).do(0x170ac6410c60, 0x170ac619fcc0)\n    \t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac699d338 sp=0x170ac699d148 pc=0x22af817\n    net/http.(*Client).Do(...)\n    \t/usr/lib/golang/src/net/http/client.go:592\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e000}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e36b0}, 0x170ac65f7e00, {0x170ac65d9680, 0x118})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac699d630 sp=0x170ac699d338 pc=0x38de6bd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e000}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x170ac65f7da0, 0x0, {0x0, ...}, ...})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac699d888 sp=0x170ac699d630 pc=0x38ddecd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e000?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de1a48?}, 0x3?, 0x2022a4d?)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac699d960 sp=0x170ac699d888 pc=0x38dd40b\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194850?}, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x170ac5de1a48, 0x3, 0x4})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac699d9e0 sp=0x170ac699d960 pc=0x38e51e2\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194850}, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x0?, 0x2?, 0x0?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac699da78 sp=0x170ac699d9e0 pc=0x38e4eaf\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac60207c0}, 0x170ac65fa000)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac699dd60 sp=0x170ac699da78 pc=0x391b5c5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac60207c0?}, 0x0?)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac699de88 sp=0x170ac699dd60 pc=0x3929c16\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac699df78 sp=0x170ac699de88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac699dfe0 sp=0x170ac699df78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac699dfe8 sp=0x170ac699dfe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 107 gp=0x170ac61d45a0 m=nil [select]:\n    runtime.gopark(0x170ac69b2ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69b29d8?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69b2848 sp=0x170ac69b2828 pc=0x1f38d2e\n    runtime.selectgo(0x170ac69b2ac8, 0x170ac69b29d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69b2978 sp=0x170ac69b2848 pc=0x1f152a5\n    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e500, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69b2b40 sp=0x170ac69b2978 pc=0x22d92e5\n    net/http.(*http2ClientConn).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8603\n    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e500, {0x4c?, 0xfa?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69b2c80 sp=0x170ac69b2b40 pc=0x22d50ad\n    net/http.(*http2Transport).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7852\n    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e500?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69b2cc0 sp=0x170ac69b2c80 pc=0x22e3a9c\n    net/http.(*Transport).roundTrip(0x8175400, 0x170ac699e3c0)\n    \t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69b2ea0 sp=0x170ac69b2cc0 pc=0x231037f\n    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n    \t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69b2ec0 sp=0x170ac69b2ea0 pc=0x2321178\n    net/http.send(0x170ac699e3c0, {0x59a1ce0, 0x8175400}, {0x170ac69b3138?, 0x1f3bac6?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69b30b0 sp=0x170ac69b2ec0 pc=0x22ad9ab\n    net/http.(*Client).send(0x170ac6410c60, 0x170ac699e3c0, {0x4d3aa8?, 0xcd6610?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69b3148 sp=0x170ac69b30b0 pc=0x22ad1d8\n    net/http.(*Client).do(0x170ac6410c60, 0x170ac699e3c0)\n    \t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69b3338 sp=0x170ac69b3148 pc=0x22af817\n    net/http.(*Client).Do(...)\n    \t/usr/lib/golang/src/net/http/client.go:592\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e1e0}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3a60}, 0x170ac69a0120, {0x170ac65d98c0, 0x111})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69b3630 sp=0x170ac69b3338 pc=0x38de6bd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e1e0}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x170ac69a00c0, 0x0, {0x0, ...}, ...})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69b3888 sp=0x170ac69b3630 pc=0x38ddecd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e1e0?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de2a48?}, 0x3?, 0x2022a4d?)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69b3960 sp=0x170ac69b3888 pc=0x38dd40b\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194bd0?}, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x170ac5de2a48, 0x3, 0x4})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69b39e0 sp=0x170ac69b3960 pc=0x38e51e2\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194bd0}, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x0?, 0x2?, 0x0?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69b3a78 sp=0x170ac69b39e0 pc=0x38e4eaf\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020ec0}, 0x170ac65fa000)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69b3d60 sp=0x170ac69b3a78 pc=0x391b5c5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020ec0?}, 0x0?)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69b3e88 sp=0x170ac69b3d60 pc=0x3929c16\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69b3f78 sp=0x170ac69b3e88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69b3fe0 sp=0x170ac69b3f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69b3fe8 sp=0x170ac69b3fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 106 gp=0x170ac648b680 m=nil [select]:\n    runtime.gopark(0x170ac69aaac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69aa9d8?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69aa848 sp=0x170ac69aa828 pc=0x1f38d2e\n    runtime.selectgo(0x170ac69aaac8, 0x170ac69aa9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69aa978 sp=0x170ac69aa848 pc=0x1f152a5\n    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e280, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69aab40 sp=0x170ac69aa978 pc=0x22d92e5\n    net/http.(*http2ClientConn).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8603\n    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e280, {0x4c?, 0xfa?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69aac80 sp=0x170ac69aab40 pc=0x22d50ad\n    net/http.(*http2Transport).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7852\n    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e280?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69aacc0 sp=0x170ac69aac80 pc=0x22e3a9c\n    net/http.(*Transport).roundTrip(0x8175400, 0x170ac699e140)\n    \t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69aaea0 sp=0x170ac69aacc0 pc=0x231037f\n    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n    \t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69aaec0 sp=0x170ac69aaea0 pc=0x2321178\n    net/http.send(0x170ac699e140, {0x59a1ce0, 0x8175400}, {0x170ac69ab138?, 0x1f3bac6?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69ab0b0 sp=0x170ac69aaec0 pc=0x22ad9ab\n    net/http.(*Client).send(0x170ac6410c60, 0x170ac699e140, {0x170ac69ab1e8?, 0x1fdd787?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69ab148 sp=0x170ac69ab0b0 pc=0x22ad1d8\n    net/http.(*Client).do(0x170ac6410c60, 0x170ac699e140)\n    \t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69ab338 sp=0x170ac69ab148 pc=0x22af817\n    net/http.(*Client).Do(...)\n    \t/usr/lib/golang/src/net/http/client.go:592\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e0f0}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e38c0}, 0x170ac69a0030, {0x170ac65d97a0, 0x10e})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69ab630 sp=0x170ac69ab338 pc=0x38de6bd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e0f0}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x170ac65f7ec0, 0x0, {0x0, ...}, ...})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69ab888 sp=0x170ac69ab630 pc=0x38ddecd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e0f0?}, 0xa8?, {0x170ac5db1a50?, 0x170ac6766a48?}, 0x3?, 0x2022a4d?)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69ab960 sp=0x170ac69ab888 pc=0x38dd40b\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194a80?}, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x170ac6766a48, 0x3, 0x4})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69ab9e0 sp=0x170ac69ab960 pc=0x38e51e2\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194a80}, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x0?, 0x2?, 0x0?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69aba78 sp=0x170ac69ab9e0 pc=0x38e4eaf\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020cc0}, 0x170ac65fa000)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69abd60 sp=0x170ac69aba78 pc=0x391b5c5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac6766ea0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020cc0?}, 0x170ac5f6f3b0?)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69abe88 sp=0x170ac69abd60 pc=0x3929c16\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69abf78 sp=0x170ac69abe88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69abfe0 sp=0x170ac69abf78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69abfe8 sp=0x170ac69abfe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 108 gp=0x170ac61d4780 m=nil [select]:\n    runtime.gopark(0x170ac69c0ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69c09d8?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69c0848 sp=0x170ac69c0828 pc=0x1f38d2e\n    runtime.selectgo(0x170ac69c0ac8, 0x170ac69c09d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69c0978 sp=0x170ac69c0848 pc=0x1f152a5\n    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e780, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69c0b40 sp=0x170ac69c0978 pc=0x22d92e5\n    net/http.(*http2ClientConn).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8603\n    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e780, {0x4c?, 0xfa?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69c0c80 sp=0x170ac69c0b40 pc=0x22d50ad\n    net/http.(*http2Transport).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7852\n    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e780?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69c0cc0 sp=0x170ac69c0c80 pc=0x22e3a9c\n    net/http.(*Transport).roundTrip(0x8175400, 0x170ac699e640)\n    \t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69c0ea0 sp=0x170ac69c0cc0 pc=0x231037f\n    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n    \t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69c0ec0 sp=0x170ac69c0ea0 pc=0x2321178\n    net/http.send(0x170ac699e640, {0x59a1ce0, 0x8175400}, {0x170ac69c1138?, 0x1f3bac6?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69c10b0 sp=0x170ac69c0ec0 pc=0x22ad9ab\n    net/http.(*Client).send(0x170ac6410c60, 0x170ac699e640, {0x82bed80?, 0x6401fdd700?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69c1148 sp=0x170ac69c10b0 pc=0x22ad1d8\n    net/http.(*Client).do(0x170ac6410c60, 0x170ac699e640)\n    \t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69c1338 sp=0x170ac69c1148 pc=0x22af817\n    net/http.(*Client).Do(...)\n    \t/usr/lib/golang/src/net/http/client.go:592\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e280}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3bf0}, 0x170ac69a0210, {0x170ac65d99e0, 0x110})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69c1630 sp=0x170ac69c1338 pc=0x38de6bd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e280}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x170ac69a01b0, 0x0, {0x0, ...}, ...})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69c1888 sp=0x170ac69c1630 pc=0x38ddecd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e280?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de2a48?}, 0x3?, 0x2022a4d?)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69c1960 sp=0x170ac69c1888 pc=0x38dd40b\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194d90?}, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x170ac5de2a48, 0x3, 0x4})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69c19e0 sp=0x170ac69c1960 pc=0x38e51e2\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194d90}, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x0?, 0x2?, 0x0?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69c1a78 sp=0x170ac69c19e0 pc=0x38e4eaf\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6021d80}, 0x170ac65fa000)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69c1d60 sp=0x170ac69c1a78 pc=0x391b5c5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac66e1d6a?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6021d80?}, 0x0?)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69c1e88 sp=0x170ac69c1d60 pc=0x3929c16\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69c1f78 sp=0x170ac69c1e88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69c1fe0 sp=0x170ac69c1f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69c1fe8 sp=0x170ac69c1fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 109 gp=0x170ac61d4960 m=nil [select]:\n    runtime.gopark(0x170ac69caac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69ca9d8?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69ca848 sp=0x170ac69ca828 pc=0x1f38d2e\n    runtime.selectgo(0x170ac69caac8, 0x170ac69ca9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69ca978 sp=0x170ac69ca848 pc=0x1f152a5\n    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699ea00, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69cab40 sp=0x170ac69ca978 pc=0x22d92e5\n    net/http.(*http2ClientConn).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8603\n    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699ea00, {0x4c?, 0xfa?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69cac80 sp=0x170ac69cab40 pc=0x22d50ad\n    net/http.(*http2Transport).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7852\n    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699ea00?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69cacc0 sp=0x170ac69cac80 pc=0x22e3a9c\n    net/http.(*Transport).roundTrip(0x8175400, 0x170ac699e8c0)\n    \t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69caea0 sp=0x170ac69cacc0 pc=0x231037f\n    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n    \t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69caec0 sp=0x170ac69caea0 pc=0x2321178\n    net/http.send(0x170ac699e8c0, {0x59a1ce0, 0x8175400}, {0x170ac69cb138?, 0x1f3bac6?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69cb0b0 sp=0x170ac69caec0 pc=0x22ad9ab\n    net/http.(*Client).send(0x170ac6410c60, 0x170ac699e8c0, {0x170ac69cb1e8?, 0x1fdd787?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69cb148 sp=0x170ac69cb0b0 pc=0x22ad1d8\n    net/http.(*Client).do(0x170ac6410c60, 0x170ac699e8c0)\n    \t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69cb338 sp=0x170ac69cb148 pc=0x22af817\n    net/http.(*Client).Do(...)\n    \t/usr/lib/golang/src/net/http/client.go:592\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e410}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3d80}, 0x170ac69a0300, {0x170ac65d9b00, 0x114})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69cb630 sp=0x170ac69cb338 pc=0x38de6bd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e410}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x170ac69a02a0, 0x0, {0x0, ...}, ...})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69cb888 sp=0x170ac69cb630 pc=0x38ddecd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e410?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de3a48?}, 0x3?, 0x2022a4d?)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69cb960 sp=0x170ac69cb888 pc=0x38dd40b\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194620?}, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x170ac5de3a48, 0x3, 0x4})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69cb9e0 sp=0x170ac69cb960 pc=0x38e51e2\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194620}, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x0?, 0x2?, 0x2700000000000000?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69cba78 sp=0x170ac69cb9e0 pc=0x38e4eaf\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac65f81c0}, 0x170ac65fa000)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69cbd60 sp=0x170ac69cba78 pc=0x391b5c5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac65f81c0?}, 0x0?)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69cbe88 sp=0x170ac69cbd60 pc=0x3929c16\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69cbf78 sp=0x170ac69cbe88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69cbfe0 sp=0x170ac69cbf78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69cbfe8 sp=0x170ac69cbfe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 110 gp=0x170ac61d4b40 m=nil [select]:\n    runtime.gopark(0x170ac6927f08?, 0x6?, 0x40?, 0x4b?, 0x170ac6927da4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6927c08 sp=0x170ac6927be8 pc=0x1f38d2e\n    runtime.selectgo(0x170ac6927f08, 0x170ac6927d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6927d38 sp=0x170ac6927c08 pc=0x1f152a5\n    net/http.(*http2clientStream).writeRequest(0x170ac61a0f00, 0x170ac619fb80, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6927f90 sp=0x170ac6927d38 pc=0x22da6a8\n    net/http.(*http2clientStream).doRequest(0x170ac61a0f00, 0x0?, 0x170ac66e2798?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6927fb8 sp=0x170ac6927f90 pc=0x22d99d8\n    net/http.(*http2ClientConn).roundTrip.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6927fe0 sp=0x170ac6927fb8 pc=0x22d999f\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6927fe8 sp=0x170ac6927fe0 pc=0x1f40801\n    created by net/http.(*http2ClientConn).roundTrip in goroutine 104\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n    \n    goroutine 111 gp=0x170ac61d4d20 m=nil [select]:\n    runtime.gopark(0x170ac658df08?, 0x6?, 0x0?, 0x7?, 0x170ac658dda4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac658dc08 sp=0x170ac658dbe8 pc=0x1f38d2e\n    runtime.selectgo(0x170ac658df08, 0x170ac658dd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac658dd38 sp=0x170ac658dc08 pc=0x1f152a5\n    net/http.(*http2clientStream).writeRequest(0x170ac61a1080, 0x170ac699e000, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac658df90 sp=0x170ac658dd38 pc=0x22da6a8\n    net/http.(*http2clientStream).doRequest(0x170ac61a1080, 0x0?, 0x170ac66e2798?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac658dfb8 sp=0x170ac658df90 pc=0x22d99d8\n    net/http.(*http2ClientConn).roundTrip.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac658dfe0 sp=0x170ac658dfb8 pc=0x22d999f\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac658dfe8 sp=0x170ac658dfe0 pc=0x1f40801\n    created by net/http.(*http2ClientConn).roundTrip in goroutine 105\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n    \n    goroutine 112 gp=0x170ac61d4f00 m=nil [select]:\n    runtime.gopark(0x170ac6589f08?, 0x6?, 0x0?, 0x7?, 0x170ac6589da4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6589c08 sp=0x170ac6589be8 pc=0x1f38d2e\n    runtime.selectgo(0x170ac6589f08, 0x170ac6589d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6589d38 sp=0x170ac6589c08 pc=0x1f152a5\n    net/http.(*http2clientStream).writeRequest(0x170ac61a1200, 0x170ac699e280, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6589f90 sp=0x170ac6589d38 pc=0x22da6a8\n    net/http.(*http2clientStream).doRequest(0x170ac61a1200, 0x0?, 0x170ac66e1f98?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6589fb8 sp=0x170ac6589f90 pc=0x22d99d8\n    net/http.(*http2ClientConn).roundTrip.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6589fe0 sp=0x170ac6589fb8 pc=0x22d999f\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6589fe8 sp=0x170ac6589fe0 pc=0x1f40801\n    created by net/http.(*http2ClientConn).roundTrip in goroutine 106\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n    \n    goroutine 113 gp=0x170ac61d50e0 m=nil [select]:\n    runtime.gopark(0x170ac658ff08?, 0x6?, 0x0?, 0x7?, 0x170ac658fda4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac658fc08 sp=0x170ac658fbe8 pc=0x1f38d2e\n    runtime.selectgo(0x170ac658ff08, 0x170ac658fd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac658fd38 sp=0x170ac658fc08 pc=0x1f152a5\n    net/http.(*http2clientStream).writeRequest(0x170ac61a1380, 0x170ac699e780, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac658ff90 sp=0x170ac658fd38 pc=0x22da6a8\n    net/http.(*http2clientStream).doRequest(0x170ac61a1380, 0x0?, 0x170ac66e1f98?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac658ffb8 sp=0x170ac658ff90 pc=0x22d99d8\n    net/http.(*http2ClientConn).roundTrip.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac658ffe0 sp=0x170ac658ffb8 pc=0x22d999f\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac658ffe8 sp=0x170ac658ffe0 pc=0x1f40801\n    created by net/http.(*http2ClientConn).roundTrip in goroutine 108\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n    \n    goroutine 114 gp=0x170ac61d52c0 m=nil [select]:\n    runtime.gopark(0x170ac6923f08?, 0x6?, 0x0?, 0x7?, 0x170ac6923da4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6923c08 sp=0x170ac6923be8 pc=0x1f38d2e\n    runtime.selectgo(0x170ac6923f08, 0x170ac6923d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6923d38 sp=0x170ac6923c08 pc=0x1f152a5\n    net/http.(*http2clientStream).writeRequest(0x170ac61a1500, 0x170ac699ea00, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6923f90 sp=0x170ac6923d38 pc=0x22da6a8\n    net/http.(*http2clientStream).doRequest(0x170ac61a1500, 0x0?, 0x170ac66e1f98?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6923fb8 sp=0x170ac6923f90 pc=0x22d99d8\n    net/http.(*http2ClientConn).roundTrip.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6923fe0 sp=0x170ac6923fb8 pc=0x22d999f\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6923fe8 sp=0x170ac6923fe0 pc=0x1f40801\n    created by net/http.(*http2ClientConn).roundTrip in goroutine 109\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n    \n    goroutine 115 gp=0x170ac61d54a0 m=nil [select]:\n    runtime.gopark(0x170ac6925f08?, 0x6?, 0x38?, 0x5b?, 0x170ac6925da4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6925c08 sp=0x170ac6925be8 pc=0x1f38d2e\n    runtime.selectgo(0x170ac6925f08, 0x170ac6925d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6925d38 sp=0x170ac6925c08 pc=0x1f152a5\n    net/http.(*http2clientStream).writeRequest(0x170ac61a1680, 0x170ac699e500, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6925f90 sp=0x170ac6925d38 pc=0x22da6a8\n    net/http.(*http2clientStream).doRequest(0x170ac61a1680, 0x0?, 0x170ac66e1f98?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6925fb8 sp=0x170ac6925f90 pc=0x22d99d8\n    net/http.(*http2ClientConn).roundTrip.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6925fe0 sp=0x170ac6925fb8 pc=0x22d999f\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6925fe8 sp=0x170ac6925fe0 pc=0x1f40801\n    created by net/http.(*http2ClientConn).roundTrip in goroutine 107\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n    \n    rax    0x0\n    rbx    0x7f7eabfff640\n    rcx    0x7f7ef9baa94c\n    rdx    0x6\n    rdi    0x10a2e\n    rsi    0x10a30\n    rbp    0x10a30\n    rsp    0x7f7eabffe790\n    r8     0x7f7eabffe860\n    r9     0x7f7ef9cd04e0\n    r10    0x8\n    r11    0x246\n    r12    0x6\n    r13    0x7f7eabffeb60\n    r14    0x170ac5c1f0e0\n    r15    0xffffffffffffffff\n    rip    0x7f7ef9baa94c\n    rflags 0x246\n    cs     0x33\n    fs     0x0\n    gs     0x0\n    \n      query-infra-both: command failed: exit status 2\n    output: runtime/cgo: pthread_create failed: Resource temporarily unavailable\n    SIGABRT: abort\n    PC=0x7f5e9c16994c m=2 sigcode=18446744073709551610\n    \n    goroutine 0 gp=0x280029a48b40 m=2 mp=0x280029abb008 [idle]:\n    runtime: g 0 gp=0x280029a48b40: unknown pc 0x7f5e9c16994c\n    stack: frame={sp:0x7f5e554a7740, fp:0x0} stack=[0x7f5e54ca81e0,0x7f5e554a7de0)\n                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n    00007f5e554a7640: 00000000 01eb64e5  00000000 00000000  .d..............\n                      ^ \u003cinternal/runtime/syscall/linux.EpollWait+0x45\u003e\n    00007f5e554a7650: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7660: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7670: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7680: 00007f5e 554a76d4  00007f5e 554a7d18  .vJU^....}JU^...\n    00007f5e554a7690: 00000000 01efa894  00000000 00000000  ................\n                      ^ \u003cruntime.netpoll+0xd4\u003e\n    00007f5e554a76a0: 00007f5e 554a76d4  00000000 00000000  .vJU^...........\n    00007f5e554a76b0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a76c0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a76d0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a76e0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a76f0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7700: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7710: 00000000 00000000  00007f5e 9c176fa9  .........o..^...\n    00007f5e554a7720: 00000000 00000000  00000000 00000011  ................\n    00007f5e554a7730: 00000000 00000130  00007f5e 9c16993e  0.......\u003e...^...\n    00007f5e554a7740: 00000000 00000120  00000000 00000000   ...............\n                      ^ SP\n    00007f5e554a7750: 00000000 00000013  00000000 00000000  ................\n    00007f5e554a7760: 00000000 00000000  00007f5e 9c1eccff  ............^...\n    00007f5e554a7770: 00007f5e 327fc910  00007f5e 9c1eccb1  ...2^.......^...\n    00007f5e554a7780: 00007f5e 327fc640  00007f5e 554a7a50  @..2^...PzJU^...\n    00007f5e554a7790: 00007f5e 554a78be  00007f5e 554a78bf  .xJU^....xJU^...\n    00007f5e554a77a0: 00007f5e 327fc640  00007f5e 9c1677f5  @..2^....w..^...\n    00007f5e554a77b0: 00007f5e 50000020  00000000 00000120   ..P^... .......\n    00007f5e554a77c0: 00000000 003d0f00  5a23f32c fe4c8400  ..=.......L.,.#Z\n    00007f5e554a77d0: 00007f5e 554a8640  00000000 00000006  @.JU^...........\n    00007f5e554a77e0: 00000000 0bac42a0  00007f5e 554a7b10  .B.......{JU^...\n    00007f5e554a77f0: 00002800 29a48b40  00007f5e 9c11c646  @..).(..F...^...\n    00007f5e554a7800: 00007f5e 9c2d9e90  00007f5e 9c1067f3  ..-.^....g..^...\n    00007f5e554a7810: 00000000 00000020  5a23f32c fe4c8400   .........L.,.#Z\n    00007f5e554a7820: 00007f5e 31ffc000  00007f5e 9c1dbac4  ...1^.......^...\n    00007f5e554a7830: 00000000 00000000  00000000 0000000d  ................\n    runtime: g 0 gp=0x280029a48b40: unknown pc 0x7f5e9c16994c\n    stack: frame={sp:0x7f5e554a7740, fp:0x0} stack=[0x7f5e54ca81e0,0x7f5e554a7de0)\n                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n    00007f5e554a7640: 00000000 01eb64e5  00000000 00000000  .d..............\n                      ^ \u003cinternal/runtime/syscall/linux.EpollWait+0x45\u003e\n    00007f5e554a7650: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7660: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7670: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7680: 00007f5e 554a76d4  00007f5e 554a7d18  .vJU^....}JU^...\n    00007f5e554a7690: 00000000 01efa894  00000000 00000000  ................\n                      ^ \u003cruntime.netpoll+0xd4\u003e\n    00007f5e554a76a0: 00007f5e 554a76d4  00000000 00000000  .vJU^...........\n    00007f5e554a76b0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a76c0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a76d0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a76e0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a76f0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7700: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7710: 00000000 00000000  00007f5e 9c176fa9  .........o..^...\n    00007f5e554a7720: 00000000 00000000  00000000 00000011  ................\n    00007f5e554a7730: 00000000 00000130  00007f5e 9c16993e  0.......\u003e...^...\n    00007f5e554a7740: 00000000 00000120  00000000 00000000   ...............\n                      ^ SP\n    00007f5e554a7750: 00000000 00000013  00000000 00000000  ................\n    00007f5e554a7760: 00000000 00000000  00007f5e 9c1eccff  ............^...\n    00007f5e554a7770: 00007f5e 327fc910  00007f5e 9c1eccb1  ...2^.......^...\n    00007f5e554a7780: 00007f5e 327fc640  00007f5e 554a7a50  @..2^...PzJU^...\n    00007f5e554a7790: 00007f5e 554a78be  00007f5e 554a78bf  .xJU^....xJU^...\n    00007f5e554a77a0: 00007f5e 327fc640  00007f5e 9c1677f5  @..2^....w..^...\n    00007f5e554a77b0: 00007f5e 50000020  00000000 00000120   ..P^... .......\n    00007f5e554a77c0: 00000000 003d0f00  5a23f32c fe4c8400  ..=.......L.,.#Z\n    00007f5e554a77d0: 00007f5e 554a8640  00000000 00000006  @.JU^...........\n    00007f5e554a77e0: 00000000 0bac42a0  00007f5e 554a7b10  .B.......{JU^...\n    00007f5e554a77f0: 00002800 29a48b40  00007f5e 9c11c646  @..).(..F...^...\n    00007f5e554a7800: 00007f5e 9c2d9e90  00007f5e 9c1067f3  ..-.^....g..^...\n    00007f5e554a7810: 00000000 00000020  5a23f32c fe4c8400   .........L.,.#Z\n    00007f5e554a7820: 00007f5e 31ffc000  00007f5e 9c1dbac4  ...1^.......^...\n    00007f5e554a7830: 00000000 00000000  00000000 0000000d  ................\n    \n    goroutine 1 gp=0x280029a481e0 m=nil [sync.WaitGroup.Wait]:\n    runtime.gopark(0x28002a5927b8?, 0x2022a4d?, 0x30?, 0xe9?, 0x28002a5927e8?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a5927a0 sp=0x28002a592780 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x280029ae34d0, 0x0, 0x1, 0x0, 0x19)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a592808 sp=0x28002a5927a0 pc=0x1f162b2\n    sync.runtime_SemacquireWaitGroup(0x28002a83b578?, 0xe0?)\n    \t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x28002a592840 sp=0x28002a592808 pc=0x1f3a66e\n    sync.(*WaitGroup).Wait(0x280029ae34c8)\n    \t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x28002a592868 sp=0x28002a592840 pc=0x1f4f445\n    golang.org/x/sync/errgroup.(*Group).Wait(0x280029ae34c0)\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x28002a592888 sp=0x28002a592868 pc=0x362507e\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0}, {0x28002a977b50, 0x1, 0x1})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x28002a592948 sp=0x28002a592888 pc=0x3927e05\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).gatherInfraLogs(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:587 +0x867 fp=0x28002a592d30 sp=0x28002a592948 pc=0x3927727\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:333 +0x533 fp=0x28002a593908 sp=0x28002a592d30 pc=0x3921bf3\n    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedInfraQueryOptions).RunInfra(0x28002a11e1e0, {0x59f0b08, 0x28002a1873b0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:83 +0x71f fp=0x28002a593b30 sp=0x28002a593908 pc=0x393887f\n    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawInfraQueryOptions).Run(0x0?, {0x59f0b08, 0x28002a1873b0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_options.go:109 +0x54 fp=0x28002a593b58 sp=0x28002a593b30 pc=0x3939134\n    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryInfraCommand.func1(0x28002a53d508?, {0x280029ca3b00?, 0x4?, 0x553b0e2?})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:45 +0x28 fp=0x28002a593b80 sp=0x28002a593b58 pc=0x3938128\n    github.com/spf13/cobra.(*Command).execute(0x28002a53d508, {0x280029ab8120, 0xc, 0xc})\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x28002a593d28 sp=0x28002a593b80 pc=0x212e614\n    github.com/spf13/cobra.(*Command).ExecuteC(0x280029ec9b08)\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x28002a593e18 sp=0x28002a593d28 pc=0x212efa5\n    github.com/spf13/cobra.(*Command).Execute(...)\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071\n    github.com/spf13/cobra.(*Command).ExecuteContext(...)\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064\n    main.main()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x28002a593f48 sp=0x28002a593e18 pc=0x47ddd6a\n    runtime.main()\n    \t/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x28002a593fe0 sp=0x28002a593f48 pc=0x1f01975\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a593fe8 sp=0x28002a593fe0 pc=0x1f40801\n    \n    goroutine 2 gp=0x280029a48d20 m=nil [force gc (idle)]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b08fa8 sp=0x280029b08f88 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.forcegchelper()\n    \t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x280029b08fe0 sp=0x280029b08fa8 pc=0x1f01c93\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b08fe8 sp=0x280029b08fe0 pc=0x1f40801\n    created by runtime.init.7 in goroutine 1\n    \t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\n    \n    goroutine 3 gp=0x280029a492c0 m=nil [GC sweep wait]:\n    runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b09788 sp=0x280029b09768 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.bgsweep(0x280029b30000)\n    \t/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x280029b097c8 sp=0x280029b09788 pc=0x1ee9f11\n    runtime.gcenable.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x280029b097e0 sp=0x280029b097c8 pc=0x1edadd7\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b097e8 sp=0x280029b097e0 pc=0x1f40801\n    created by runtime.gcenable in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\n    \n    goroutine 4 gp=0x280029a494a0 m=nil [GC scavenge wait]:\n    runtime.gopark(0x10000?, 0x5981cd8?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b09f78 sp=0x280029b09f58 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.(*scavengerState).park(0x82979c0)\n    \t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x280029b09fa8 sp=0x280029b09f78 pc=0x1ee77e9\n    runtime.bgscavenge(0x280029b30000)\n    \t/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x280029b09fc8 sp=0x280029b09fa8 pc=0x1ee7d99\n    runtime.gcenable.gowrap2()\n    \t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x280029b09fe0 sp=0x280029b09fc8 pc=0x1edad97\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b09fe8 sp=0x280029b09fe0 pc=0x1f40801\n    created by runtime.gcenable in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\n    \n    goroutine 5 gp=0x280029a49a40 m=nil [GOMAXPROCS updater (idle)]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0a788 sp=0x280029b0a768 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.updateMaxProcsGoroutine()\n    \t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x280029b0a7e0 sp=0x280029b0a788 pc=0x1f10267\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0a7e8 sp=0x280029b0a7e0 pc=0x1f40801\n    created by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\n    \t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\n    \n    goroutine 6 gp=0x280029b481e0 m=nil [finalizer wait]:\n    runtime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x280029b08601?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b08620 sp=0x280029b08600 pc=0x1f38d2e\n    runtime.runFinalizers()\n    \t/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x280029b087e0 sp=0x280029b08620 pc=0x1ed9cc7\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b087e8 sp=0x280029b087e0 pc=0x1f40801\n    created by runtime.createfing in goroutine 1\n    \t/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d\n    \n    goroutine 7 gp=0x280029d3fc20 m=nil [cleanup wait]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0af68 sp=0x280029b0af48 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.(*cleanupQueue).dequeue(0x82980e0)\n    \t/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x280029b0afa0 sp=0x280029b0af68 pc=0x1ed6074\n    runtime.runCleanups()\n    \t/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x280029b0afe0 sp=0x280029b0afa0 pc=0x1ed66e5\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0afe8 sp=0x280029b0afe0 pc=0x1f40801\n    created by runtime.(*cleanupQueue).createGs in goroutine 1\n    \t/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5\n    \n    goroutine 8 gp=0x280029ee6000 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125c86ae32d7?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0b740 sp=0x280029b0b720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x280029ad7ab0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x280029b0b7c8 sp=0x280029b0b740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b0b7e0 sp=0x280029b0b7c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0b7e8 sp=0x280029b0b7e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 18 gp=0x280029b843c0 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125c662e5759?, 0x3?, 0xc2?, 0x10?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b04740 sp=0x280029b04720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x280029ad7ab0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x280029b047c8 sp=0x280029b04740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b047e0 sp=0x280029b047c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b047e8 sp=0x280029b047e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 9 gp=0x280029ee61e0 m=nil [semacquire]:\n    runtime.gopark(0x6e2b22?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029b0bc96?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0be38 sp=0x280029b0be18 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029b0bea0 sp=0x280029b0be38 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029b0bf40 sp=0x280029b0bea0 pc=0x1edbb12\n    runtime.gcBgMarkWorker(0x280029ad7ab0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029b0bfc8 sp=0x280029b0bf40 pc=0x1edda88\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b0bfe0 sp=0x280029b0bfc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0bfe8 sp=0x280029b0bfe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 10 gp=0x280029ee63c0 m=nil [semacquire]:\n    runtime.gopark(0x1ec6f5e?, 0x1f3ec12?, 0x80?, 0x51?, 0x1ec70df?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eec638 sp=0x280029eec618 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eec6a0 sp=0x280029eec638 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eec740 sp=0x280029eec6a0 pc=0x1edbb12\n    runtime.gcBgMarkWorker(0x280029ad7ab0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eec7c8 sp=0x280029eec740 pc=0x1edda88\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eec7e0 sp=0x280029eec7c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eec7e8 sp=0x280029eec7e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 11 gp=0x280029ee65a0 m=nil [semacquire]:\n    runtime.gopark(0x54b522?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029eecc96?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eece38 sp=0x280029eece18 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eecea0 sp=0x280029eece38 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eecf40 sp=0x280029eecea0 pc=0x1edbb12\n    runtime.gcBgMarkWorker(0x280029ad7ab0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eecfc8 sp=0x280029eecf40 pc=0x1edda88\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eecfe0 sp=0x280029eecfc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eecfe8 sp=0x280029eecfe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 34 gp=0x280029f04000 m=nil [semacquire]:\n    runtime.gopark(0x2d7122?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029ee8496?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029ee8638 sp=0x280029ee8618 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029ee86a0 sp=0x280029ee8638 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029ee8740 sp=0x280029ee86a0 pc=0x1edbb12\n    runtime.gcBgMarkWorker(0x280029ad7ab0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029ee87c8 sp=0x280029ee8740 pc=0x1edda88\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029ee87e0 sp=0x280029ee87c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029ee87e8 sp=0x280029ee87e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 19 gp=0x280029b845a0 m=4 mp=0x280029b34008 [GC mark termination]:\n    runtime.systemstack_switch()\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x280029b04ea0 sp=0x280029b04e90 pc=0x1f3ebc8\n    runtime.forEachP(...)\n    \t/usr/lib/golang/src/runtime/proc.go:2112\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x280029b04f40 sp=0x280029b04ea0 pc=0x1edbc34\n    runtime.gcBgMarkWorker(0x280029ad7ab0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029b04fc8 sp=0x280029b04f40 pc=0x1edda88\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b04fe0 sp=0x280029b04fc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b04fe8 sp=0x280029b04fe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 12 gp=0x280029ee6780 m=nil [semacquire]:\n    runtime.gopark(0x3f9e22?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029eed496?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eed638 sp=0x280029eed618 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eed6a0 sp=0x280029eed638 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eed740 sp=0x280029eed6a0 pc=0x1edbb12\n    runtime.gcBgMarkWorker(0x280029ad7ab0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eed7c8 sp=0x280029eed740 pc=0x1edda88\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eed7e0 sp=0x280029eed7c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eed7e8 sp=0x280029eed7e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 35 gp=0x280029ee74a0 m=nil [select, locked to thread]:\n    runtime.gopark(0x280029eef7a8?, 0x2?, 0x1?, 0x74?, 0x280029eef794?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eef628 sp=0x280029eef608 pc=0x1f38d2e\n    runtime.selectgo(0x280029eef7a8, 0x280029eef790, 0x0?, 0x0, 0x0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x280029eef758 sp=0x280029eef628 pc=0x1f152a5\n    runtime.ensureSigM.func1()\n    \t/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x280029eef7e0 sp=0x280029eef758 pc=0x1f328e8\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eef7e8 sp=0x280029eef7e0 pc=0x1f40801\n    created by runtime.ensureSigM in goroutine 1\n    \t/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5\n    \n    goroutine 50 gp=0x280029f84960 m=9 mp=0x28002a000008 [syscall]:\n    runtime.notetsleepg(0x82bdea0, 0xffffffffffffffff)\n    \t/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x280029ee9fa0 sp=0x280029ee9f78 pc=0x1ece509\n    os/signal.signal_recv()\n    \t/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x280029ee9fc0 sp=0x280029ee9fa0 pc=0x1f3b298\n    os/signal.loop()\n    \t/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x280029ee9fe0 sp=0x280029ee9fc0 pc=0x2030553\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029ee9fe8 sp=0x280029ee9fe0 pc=0x1f40801\n    created by os/signal.Notify.func1.1 in goroutine 1\n    \t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\n    \n    goroutine 51 gp=0x280029f84b40 m=nil [select]:\n    runtime.gopark(0x280029eea798?, 0x2?, 0x80?, 0xa3?, 0x280029eea78c?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eea628 sp=0x280029eea608 pc=0x1f38d2e\n    runtime.selectgo(0x280029eea798, 0x280029eea788, 0x0?, 0x0, 0x0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x280029eea758 sp=0x280029eea628 pc=0x1f152a5\n    os/signal.NotifyContext.func1()\n    \t/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x280029eea7e0 sp=0x280029eea758 pc=0x202ff9c\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eea7e8 sp=0x280029eea7e0 pc=0x1f40801\n    created by os/signal.NotifyContext in goroutine 1\n    \t/usr/lib/golang/src/os/signal/signal.go:291 +0x158\n    \n    goroutine 137 gp=0x28002a60a000 m=nil [sync.WaitGroup.Wait]:\n    runtime.gopark(0x28002a603e00?, 0x1ed0765?, 0x80?, 0xea?, 0x7f5e54234a40?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a603da0 sp=0x28002a603d80 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x280029ae3b10, 0x0, 0x1, 0x0, 0x19)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a603e08 sp=0x28002a603da0 pc=0x1f162b2\n    sync.runtime_SemacquireWaitGroup(0x28002a83b590?, 0x0?)\n    \t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x28002a603e40 sp=0x28002a603e08 pc=0x1f3a66e\n    sync.(*WaitGroup).Wait(0x280029ae3b08)\n    \t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x28002a603e68 sp=0x28002a603e40 pc=0x1f4f445\n    golang.org/x/sync/errgroup.(*Group).Wait(0x280029ae3b00)\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x28002a603e88 sp=0x28002a603e68 pc=0x362507e\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x280029d80e80, {0x59f0b40, 0x280029beca00}, {0x28002a977b50, 0x1, 0x38d788c?}, 0x28002a830c40)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x28002a603f00 sp=0x28002a603e88 pc=0x392976a\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x28002a603f78 sp=0x28002a603f00 pc=0x392808e\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a603fe0 sp=0x28002a603f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a603fe8 sp=0x28002a603fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 38 gp=0x28002a60b0e0 m=nil [IO wait]:\n    runtime.gopark(0x28002a7c22c0?, 0x280029aee7f8?, 0x40?, 0x77?, 0xb?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b197b8 sp=0x280029b19798 pc=0x1f38d2e\n    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n    \t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x280029b197f0 sp=0x280029b197b8 pc=0x1ef9e57\n    internal/poll.runtime_pollWait(0x7f5e554b6c00, 0x72)\n    \t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x280029b19810 sp=0x280029b197f0 pc=0x1f375c5\n    internal/poll.(*pollDesc).wait(0x280029bf6400?, 0x28002a90c000?, 0x0)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x280029b19838 sp=0x280029b19810 pc=0x1fbd167\n    internal/poll.(*pollDesc).waitRead(...)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\n    internal/poll.(*FD).Read(0x280029bf6400, {0x28002a90c000, 0x3500, 0x3500})\n    \t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x280029b198d0 sp=0x280029b19838 pc=0x1fbe4ce\n    net.(*netFD).Read(0x280029bf6400, {0x28002a90c000?, 0x28002a90c000?, 0x5?})\n    \t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x280029b19918 sp=0x280029b198d0 pc=0x20798c5\n    net.(*conn).Read(0x280029a46378, {0x28002a90c000?, 0x7f5e554a9828?, 0x7f5e9c0b1e00?})\n    \t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x280029b19960 sp=0x280029b19918 pc=0x208b7a5\n    crypto/tls.(*atLeastReader).Read(0x28002aa26ba0, {0x28002a90c000?, 0x28002a0d8658?, 0x1efcc28?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x280029b199a8 sp=0x280029b19960 pc=0x2222ddb\n    bytes.(*Buffer).ReadFrom(0x28002a0d8628, {0x59a3680, 0x28002aa26ba0})\n    \t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x280029b19a00 sp=0x280029b199a8 pc=0x1fe9258\n    crypto/tls.(*Conn).readFromUntil(0x28002a0d8388, {0x59a32c0, 0x280029a46378}, 0x222764c?)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x280029b19a38 sp=0x280029b19a00 pc=0x2222fbe\n    crypto/tls.(*Conn).readRecordOrCCS(0x28002a0d8388, 0x0)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x280029b19c18 sp=0x280029b19a38 pc=0x222009b\n    crypto/tls.(*Conn).readRecord(...)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:588\n    crypto/tls.(*Conn).Read(0x28002a0d8388, {0x280029fed000, 0x1000, 0x22c34e0?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x280029b19c80 sp=0x280029b19c18 pc=0x22267c5\n    bufio.(*Reader).Read(0x28002a7b89c0, {0x28002a7c8584, 0x9, 0x22e100e?})\n    \t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0x280029b19cb8 sp=0x280029b19c80 pc=0x20a71f7\n    io.ReadAtLeast({0x59a1260, 0x28002a7b89c0}, {0x28002a7c8584, 0x9, 0x9}, 0x9)\n    \t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0x280029b19d00 sp=0x280029b19cb8 pc=0x1fb17ee\n    io.ReadFull(...)\n    \t/usr/lib/golang/src/io/io.go:354\n    net/http.http2readFrameHeader({0x28002a7c8584, 0x9, 0x280000000015?}, {0x59a1260?, 0x28002a7b89c0?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:1805 +0x65 fp=0x280029b19d50 sp=0x280029b19d00 pc=0x22b9e25\n    net/http.(*http2Framer).ReadFrameHeader(0x28002a7c8540)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:2071 +0x6b fp=0x280029b19dd8 sp=0x280029b19d50 pc=0x22ba9eb\n    net/http.(*http2Framer).ReadFrame(0x28002a7c8540)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:2130 +0x18 fp=0x280029b19e18 sp=0x280029b19dd8 pc=0x22bb1b8\n    net/http.(*http2clientConnReadLoop).run(0x280029b19fa8)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9550 +0xca fp=0x280029b19f60 sp=0x280029b19e18 pc=0x22de14a\n    net/http.(*http2ClientConn).readLoop(0x280029bc01c0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x280029b19fc8 sp=0x280029b19f60 pc=0x22dd532\n    net/http.(*http2Transport).newClientConn.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x280029b19fe0 sp=0x280029b19fc8 pc=0x22d7057\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b19fe8 sp=0x280029b19fe0 pc=0x1f40801\n    created by net/http.(*http2Transport).newClientConn in goroutine 37\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5\n    \n    goroutine 69 gp=0x28002a60b4a0 m=nil [IO wait]:\n    runtime.gopark(0x28002a50c560?, 0x1edb176?, 0x80?, 0xed?, 0xb?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a606808 sp=0x28002a6067e8 pc=0x1f38d2e\n    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n    \t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x28002a606840 sp=0x28002a606808 pc=0x1ef9e57\n    internal/poll.runtime_pollWait(0x7f5e554b6a00, 0x72)\n    \t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x28002a606860 sp=0x28002a606840 pc=0x1f375c5\n    internal/poll.(*pollDesc).wait(0x280029bf6c80?, 0x280029a5a000?, 0x0)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x28002a606888 sp=0x28002a606860 pc=0x1fbd167\n    internal/poll.(*pollDesc).waitRead(...)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\n    internal/poll.(*FD).Read(0x280029bf6c80, {0x280029a5a000, 0x2500, 0x2500})\n    \t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x28002a606920 sp=0x28002a606888 pc=0x1fbe4ce\n    net.(*netFD).Read(0x280029bf6c80, {0x280029a5a000?, 0x28002a60690a?, 0x1edb176?})\n    \t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x28002a606968 sp=0x28002a606920 pc=0x20798c5\n    net.(*conn).Read(0x280029a463c0, {0x280029a5a000?, 0x7f5e5423eae0?, 0x7f5e9c09dde0?})\n    \t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x28002a6069b0 sp=0x28002a606968 pc=0x208b7a5\n    crypto/tls.(*atLeastReader).Read(0x28002a628318, {0x280029a5a000?, 0x28002a606a98?, 0x1efcc28?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x28002a6069f8 sp=0x28002a6069b0 pc=0x2222ddb\n    bytes.(*Buffer).ReadFrom(0x280029ba9ea8, {0x59a3680, 0x28002a628318})\n    \t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x28002a606a50 sp=0x28002a6069f8 pc=0x1fe9258\n    crypto/tls.(*Conn).readFromUntil(0x280029ba9c08, {0x59a32c0, 0x280029a463c0}, 0x222764c?)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x28002a606a88 sp=0x28002a606a50 pc=0x2222fbe\n    crypto/tls.(*Conn).readRecordOrCCS(0x280029ba9c08, 0x0)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x28002a606c68 sp=0x28002a606a88 pc=0x222009b\n    crypto/tls.(*Conn).readRecord(...)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:588\n    crypto/tls.(*Conn).Read(0x280029ba9c08, {0x28002a24a000, 0x1000, 0x4e55540?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x28002a606cd0 sp=0x28002a606c68 pc=0x22267c5\n    net/http.(*persistConn).Read(0x28002a0ad400, {0x28002a24a000?, 0x59a0ea0?, 0x81260e0?})\n    \t/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x28002a606d30 sp=0x28002a606cd0 pc=0x2319d07\n    bufio.(*Reader).fill(0x28002a8130e0)\n    \t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x28002a606d68 sp=0x28002a606d30 pc=0x20a6c63\n    bufio.(*Reader).Peek(0x28002a8130e0, 0x1)\n    \t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x28002a606d80 sp=0x28002a606d68 pc=0x20a6df3\n    net/http.(*persistConn).readLoop(0x28002a0ad400)\n    \t/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x28002a606fc8 sp=0x28002a606d80 pc=0x231a812\n    net/http.(*Transport).dialConn.gowrap2()\n    \t/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x28002a606fe0 sp=0x28002a606fc8 pc=0x2319197\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a606fe8 sp=0x28002a606fe0 pc=0x1f40801\n    created by net/http.(*Transport).dialConn in goroutine 55\n    \t/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac\n    \n    goroutine 70 gp=0x28002a60bc20 m=nil [select]:\n    runtime.gopark(0x28002a613f48?, 0x2?, 0x0?, 0xb6?, 0x28002a613ef4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a708d88 sp=0x28002a708d68 pc=0x1f38d2e\n    runtime.selectgo(0x28002a708f48, 0x28002a613ef0, 0x280029c93a80?, 0x0, 0x28002a4f7c80?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a708eb8 sp=0x28002a708d88 pc=0x1f152a5\n    net/http.(*persistConn).writeLoop(0x28002a0ad400)\n    \t/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x28002a708fc8 sp=0x28002a708eb8 pc=0x231c426\n    net/http.(*Transport).dialConn.gowrap3()\n    \t/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x28002a708fe0 sp=0x28002a708fc8 pc=0x2319157\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a708fe8 sp=0x28002a708fe0 pc=0x1f40801\n    created by net/http.(*Transport).dialConn in goroutine 55\n    \t/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe\n    \n    goroutine 138 gp=0x28002a84a780 m=nil [semacquire]:\n    runtime.gopark(0x1?, 0x1f78fa5?, 0xe0?, 0x75?, 0x2?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a2ca910 sp=0x28002a2ca8f0 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a2ca978 sp=0x28002a2ca910 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x28002a2caa18 sp=0x28002a2ca978 pc=0x1edbb12\n    runtime.gcAssistAlloc(0x28002a84a780)\n    \t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x28002a2caaa8 sp=0x28002a2caa18 pc=0x1ee06b6\n    runtime.deductAssistCredit(0x28002a2cac70?)\n    \t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x28002a2caac8 sp=0x28002a2caaa8 pc=0x1ed10a6\n    runtime.mallocgc(0x10, 0x4af39e0, 0x1)\n    \t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x28002a2caaf8 sp=0x28002a2caac8 pc=0x1f35e07\n    runtime.convTstring({0x28002a6a7728, 0x14})\n    \t/usr/lib/golang/src/runtime/iface.go:431 +0x45 fp=0x28002a2cab20 sp=0x28002a2caaf8 pc=0x1f35865\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x280029beca00}, 0x28002a830cb0, 0x280029f636e0)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:297 +0x394 fp=0x28002a2caf48 sp=0x28002a2cab20 pc=0x3920674\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x28002a2caf78 sp=0x28002a2caf48 pc=0x392800c\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a2cafe0 sp=0x28002a2caf78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a2cafe8 sp=0x28002a2cafe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 139 gp=0x28002a84a960 m=nil [select]:\n    runtime.gopark(0x28002a4bdd00?, 0x2?, 0x1?, 0xc9?, 0x28002a4bdc74?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a4bdac8 sp=0x28002a4bdaa8 pc=0x1f38d2e\n    runtime.selectgo(0x28002a4bdd00, 0x28002a4bdc70, 0x28002a3e8a50?, 0x0, 0x280029f827c8?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a4bdbf8 sp=0x28002a4bdac8 pc=0x1f152a5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x49424d554554534c?, {0x59f0b40, 0x280029beca00}, 0x28002a830c40, 0x28002a830cb0)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x28002a4bdf10 sp=0x28002a4bdbf8 pc=0x392836e\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x28002a4bdf78 sp=0x28002a4bdf10 pc=0x3927f3d\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a4bdfe0 sp=0x28002a4bdf78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a4bdfe8 sp=0x28002a4bdfe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 140 gp=0x28002a84ab40 m=nil [semacquire]:\n    runtime.gopark(0x1f6aa32?, 0x28002a34b418?, 0xc5?, 0xbd?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a34b3c0 sp=0x28002a34b3a0 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a34b428 sp=0x28002a34b3c0 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x28002a34b4c8 sp=0x28002a34b428 pc=0x1edbb12\n    runtime.gcAssistAlloc(0x28002a84ab40)\n    \t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x28002a34b558 sp=0x28002a34b4c8 pc=0x1ee06b6\n    runtime.deductAssistCredit(0x20?)\n    \t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x28002a34b578 sp=0x28002a34b558 pc=0x1ed10a6\n    runtime.mallocgc(0x1000, 0x4af32a0, 0x1)\n    \t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x28002a34b5a8 sp=0x28002a34b578 pc=0x1f35e07\n    runtime.makeslice(0x28002a34b668?, 0x1f8?, 0x1ff?)\n    \t/usr/lib/golang/src/runtime/slice.go:117 +0x49 fp=0x28002a34b5d0 sp=0x28002a34b5a8 pc=0x1f3b489\n    bufio.NewWriterSize(...)\n    \t/usr/lib/golang/src/bufio/bufio.go:599\n    bufio.NewWriter(...)\n    \t/usr/lib/golang/src/bufio/bufio.go:608\n    encoding/csv.NewWriter(...)\n    \t/usr/lib/golang/src/encoding/csv/writer.go:42\n    github.com/Azure/azure-kusto-go/azkustodata/query.(*row).String(0x812ab90?)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/row_impl.go:100 +0x18e fp=0x28002a34b6e8 sp=0x28002a34b5d0 pc=0x38cf5ee\n    fmt.(*pp).handleMethods(0x28002a24c0d0, 0x2a34b8f0?)\n    \t/usr/lib/golang/src/fmt/print.go:674 +0x2f4 fp=0x28002a34b8a8 sp=0x28002a34b6e8 pc=0x1fdd1d4\n    fmt.(*pp).printArg(0x28002a24c0d0, {0x549a3a0, 0x28002a7c7c80}, 0x76)\n    \t/usr/lib/golang/src/fmt/print.go:757 +0x42e fp=0x28002a34b938 sp=0x28002a34b8a8 pc=0x1fdd9ee\n    fmt.(*pp).doPrintf(0x28002a24c0d0, {0x553a400, 0x2}, {0x28002a34bd08, 0x1, 0x1})\n    \t/usr/lib/golang/src/fmt/print.go:1075 +0x3ec fp=0x28002a34ba20 sp=0x28002a34b938 pc=0x1fe04ec\n    fmt.Sprintf({0x553a400, 0x2}, {0x28002a34bd08, 0x1, 0x1})\n    \t/usr/lib/golang/src/fmt/print.go:239 +0x53 fp=0x28002a34ba78 sp=0x28002a34ba20 pc=0x1fda6b3\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x280029e454e0, {0x59f0b40, 0x280029beca50}, {0x5a00300, 0x280029ae2040}, 0x28002a830c40)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:160 +0x8d8 fp=0x28002a34bd60 sp=0x28002a34ba78 pc=0x391bad8\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x2d20414320746f6f?, {0x59f0b40?, 0x280029beca50?}, {0x5a00300?, 0x280029ae2040?}, 0x43437a4b4349494d?)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x28002a34be88 sp=0x28002a34bd60 pc=0x3929c16\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x28002a34bf78 sp=0x28002a34be88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a34bfe0 sp=0x28002a34bf78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a34bfe8 sp=0x28002a34bfe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 137\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 142 gp=0x28002a84af00 m=nil [select]:\n    runtime.gopark(0x28002a333de0?, 0x2?, 0x8?, 0xb8?, 0x28002a333dbc?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a333c58 sp=0x28002a333c38 pc=0x1f38d2e\n    runtime.selectgo(0x28002a333de0, 0x28002a333db8, 0x280029afc008?, 0x1, 0x1f3665f?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a333d88 sp=0x28002a333c58 pc=0x1f152a5\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.(*iterativeDataset).sendTable(0x280029becf00, {0x5a19be0, 0x28002ab876c0})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:378 +0xf3 fp=0x28002a333e10 sp=0x28002a333d88 pc=0x38d8d33\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.handleDataTable(0x280029becf00, 0x28002a3b2b40)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:324 +0x2c6 fp=0x28002a333ec8 sp=0x28002a333e10 pc=0x38d8686\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x280029becf00)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:167 +0x148 fp=0x28002a333f30 sp=0x28002a333ec8 pc=0x38d7b48\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x280029becf00, 0x28002a977da0)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x28002a333fc0 sp=0x28002a333f30 pc=0x38d788c\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x28002a333fe0 sp=0x28002a333fc0 pc=0x38d751b\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a333fe8 sp=0x28002a333fe0 pc=0x1f40801\n    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 140\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc\n    \n    rax    0x0\n    rbx    0x7f5e554a8640\n    rcx    0x7f5e9c16994c\n    rdx    0x6\n    rdi    0x10a66\n    rsi    0x10a67\n    rbp    0x10a67\n    rsp    0x7f5e554a7740\n    r8     0x7f5e554a7810\n    r9     0x7f5e9c28f4e0\n    r10    0x8\n    r11    0x246\n    r12    0x6\n    r13    0x7f5e554a7b10\n    r14    0x280029a48b40\n    r15    0xffffffffffffffff\n    rip    0x7f5e9c16994c\n    rflags 0x246\n    cs     0x33\n    fs     0x0\n    gs     0x0\n    \n    {\n        s: \"must-gather CLI test failures:\\n  query-basic: command failed: exit status 1\\noutput: \\x1b[37m[21:33:14.493]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:23.438]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\\\\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:33:21.8572874Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\\\\\\\", ActivityId:\\\\\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\\\\\", SubActivityId:\\\\\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \\\\u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:23.438]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 50/289 failed (KQL: let _startTime = datetime(2026-10-15T01:38:14.4931102Z);\\\\nlet _endTime = datetime(2026-10-15T01:43:14.4931101Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-15T01:38:14.4931102Z);\\\\nlet _endTime = datetime(2026-10-15T01:43:14.4931101Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:23.712]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:23.712]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:24.208]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 1498,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[21:33:24.208]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:24.217]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[21:33:40.606]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n  \\\"commits\\\": 793\\n}\\x1b[0m\\n\\x1b[37m[21:33:40.607]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:40.961]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:33:41.088]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j8700672-mgmt-1\\\",\\n    \\\"ci01-j8700672-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:33:41.127]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespaces\\\": [\\n    \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:33:41.127]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[21:33:41.127]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:41.317]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:41.478]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:41.623]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:43.044]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\\\\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:33:21.8572874Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\\\\\\\", ActivityId:\\\\\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\\\\\", SubActivityId:\\\\\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \\\\u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:43.044]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\\\\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:33:21.8572874Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\\\\\\\", ActivityId:\\\\\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\\\\\", SubActivityId:\\\\\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \\\\u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n  query-skip-hcp-logs: command failed: exit status 1\\noutput: \\x1b[37m[21:33:43.407]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:49.648]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\\\\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:33:47.6716888Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\\\\\\\", ActivityId:\\\\\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\\\\\", SubActivityId:\\\\\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \\\\u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:49.648]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 47/289 failed (KQL: let _startTime = datetime(2026-10-15T01:23:43.4070857Z);\\\\nlet _endTime = datetime(2026-10-15T01:28:43.4070856Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:49.744]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:49.744]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:50.271]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 1498,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[21:33:50.271]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:50.281]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[21:34:08.647]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n  \\\"commits\\\": 793\\n}\\x1b[0m\\n\\x1b[37m[21:34:08.648]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:09.212]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:34:09.309]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j8700672-mgmt-1\\\",\\n    \\\"ci01-j8700672-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:34:09.470]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespaces\\\": [\\n    \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:34:09.470]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[21:34:09.470]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:09.615]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:10.183]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:10.617]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:12.156]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\\\\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:33:47.6716888Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\\\\\\\", ActivityId:\\\\\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\\\\\", SubActivityId:\\\\\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \\\\u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:12.157]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\\\\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:33:47.6716888Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\\\\\\\", ActivityId:\\\\\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\\\\\", SubActivityId:\\\\\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \\\\u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n  query-skip-kubernetes-events: command failed: exit status 1\\noutput: \\x1b[37m[21:34:12.559]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:18.220]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\\\\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:34:16.3210435Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\\\\\\\", ActivityId:\\\\\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\\\\\", SubActivityId:\\\\\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \\\\u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:18.221]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 47/289 failed (KQL: let _startTime = datetime(2026-10-15T01:24:12.5582990Z);\\\\nlet _endTime = datetime(2026-10-15T01:29:12.5582989Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:18.221]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:18.221]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:18.723]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 1498,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[21:34:18.724]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:18.734]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[21:34:35.437]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:35.845]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:34:36.194]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j8700672-mgmt-1\\\",\\n    \\\"ci01-j8700672-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:34:36.313]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespaces\\\": [\\n    \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:34:36.313]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[21:34:36.313]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:36.544]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:36.863]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:37.093]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:38.438]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\\\\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:34:16.3210435Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\\\\\\\", ActivityId:\\\\\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\\\\\", SubActivityId:\\\\\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \\\\u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:38.438]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\\\\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:34:16.3210435Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\\\\\\\", ActivityId:\\\\\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\\\\\", SubActivityId:\\\\\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \\\\u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\"\\n}\\x1b[0m\\n\\n  query-collect-systemd-logs: command failed: exit status 1\\noutput: \\x1b[37m[21:34:38.772]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:46.539]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\\\\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:34:46.3046137Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\\\\\\\", ActivityId:\\\\\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\\\\\", SubActivityId:\\\\\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \\\\u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:46.539]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 45/289 failed (KQL: let _startTime = datetime(2026-10-15T01:14:38.7716298Z);\\\\nlet _endTime = datetime(2026-10-15T01:19:38.7716297Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:47.129]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:47.129]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:47.633]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 1498,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[21:34:47.633]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:47.642]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[21:34:59.291]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:59.911]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:35:00.221]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j8700672-mgmt-1\\\",\\n    \\\"ci01-j8700672-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:35:00.360]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespaces\\\": [\\n    \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:35:00.360]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[21:35:00.360]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:35:00.702]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:35:01.111]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:35:01.638]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:35:02.339]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\\\\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:34:46.3046137Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\\\\\\\", ActivityId:\\\\\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\\\\\", SubActivityId:\\\\\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \\\\u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:35:02.340]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\\\\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:34:46.3046137Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\\\\\\\", ActivityId:\\\\\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\\\\\", SubActivityId:\\\\\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \\\\u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\"\\n}\\x1b[0m\\n\\n  query-skip-hcp-and-events: command failed: exit status 2\\noutput: \\x1b[37m[21:35:02.599]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-1950984811\\\"\\n}\\x1b[0m\\nruntime/cgo: pthread_create failed: Resource temporarily unavailable\\nSIGABRT: abort\\nPC=0x7f80df60594c m=4 sigcode=18446744073709551610\\n\\ngoroutine 0 gp=0x14197f1334a0 m=4 mp=0x14197f21a008 [idle]:\\nruntime: g 0 gp=0x14197f1334a0: unknown pc 0x7f80df60594c\\nstack: frame={sp:0x7f808bffe790, fp:0x0} stack=[0x7f808b7ff1e0,0x7f808bffede0)\\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\\n00007f808bffe690: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6a0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6b0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6c0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6d0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6e0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6f0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe700: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe710: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe720: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe730: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe740: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe750: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe760: 00000000 00000000  00007f80 df612fa9  ........./a.....\\n00007f808bffe770: 00000000 00000000  00000000 00000011  ................\\n00007f808bffe780: 00000000 00000130  00007f80 df60593e  0.......\u003eY`.....\\n00007f808bffe790: 00000000 00000120  00000000 00000000   ...............\\n                  ^ SP\\n00007f808bffe7a0: 00010000 00000013  00000000 00000000  ................\\n00007f808bffe7b0: 00000000 00000000  00007f80 df688cff  ..........h.....\\n00007f808bffe7c0: 00007f80 89ffb910  00007f80 df688cb1  ..........h.....\\n00007f808bffe7d0: 00007f80 89ffb640  00007f80 8bffeaa0  @...............\\n00007f808bffe7e0: 00007f80 8bffe90e  00007f80 8bffe90f  ................\\n00007f808bffe7f0: 00007f80 89ffb640  00007f80 df6037f5  @........7`.....\\n00007f808bffe800: 00007f80 84000020  00000000 00000120   ....... .......\\n00007f808bffe810: 00000000 003d0f00  701e1ebb c8982900  ..=......).....p\\n00007f808bffe820: 00007f80 8bfff640  00000000 00000006  @...............\\n00007f808bffe830: 00007f80 84000c90  00007f80 8bffeb60  ........`.......\\n00007f808bffe840: 00001419 7f1334a0  00007f80 df5b8646  .4......F.[.....\\n00007f808bffe850: 00007f80 df775e90  00007f80 df5a27f3  .^w......'Z.....\\n00007f808bffe860: 00000000 00000020  701e1ebb c8982900   ........).....p\\n00007f808bffe870: 00007f80 897fb000  00007f80 df677ac4  .........zg.....\\n00007f808bffe880: 00000000 00000000  00000000 0000000d  ................\\nruntime: g 0 gp=0x14197f1334a0: unknown pc 0x7f80df60594c\\nstack: frame={sp:0x7f808bffe790, fp:0x0} stack=[0x7f808b7ff1e0,0x7f808bffede0)\\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\\n00007f808bffe690: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6a0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6b0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6c0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6d0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6e0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6f0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe700: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe710: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe720: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe730: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe740: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe750: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe760: 00000000 00000000  00007f80 df612fa9  ........./a.....\\n00007f808bffe770: 00000000 00000000  00000000 00000011  ................\\n00007f808bffe780: 00000000 00000130  00007f80 df60593e  0.......\u003eY`.....\\n00007f808bffe790: 00000000 00000120  00000000 00000000   ...............\\n                  ^ SP\\n00007f808bffe7a0: 00010000 00000013  00000000 00000000  ................\\n00007f808bffe7b0: 00000000 00000000  00007f80 df688cff  ..........h.....\\n00007f808bffe7c0: 00007f80 89ffb910  00007f80 df688cb1  ..........h.....\\n00007f808bffe7d0: 00007f80 89ffb640  00007f80 8bffeaa0  @...............\\n00007f808bffe7e0: 00007f80 8bffe90e  00007f80 8bffe90f  ................\\n00007f808bffe7f0: 00007f80 89ffb640  00007f80 df6037f5  @........7`.....\\n00007f808bffe800: 00007f80 84000020  00000000 00000120   ....... .......\\n00007f808bffe810: 00000000 003d0f00  701e1ebb c8982900  ..=......).....p\\n00007f808bffe820: 00007f80 8bfff640  00000000 00000006  @...............\\n00007f808bffe830: 00007f80 84000c90  00007f80 8bffeb60  ........`.......\\n00007f808bffe840: 00001419 7f1334a0  00007f80 df5b8646  .4......F.[.....\\n00007f808bffe850: 00007f80 df775e90  00007f80 df5a27f3  .^w......'Z.....\\n00007f808bffe860: 00000000 00000020  701e1ebb c8982900   ........).....p\\n00007f808bffe870: 00007f80 897fb000  00007f80 df677ac4  .........zg.....\\n00007f808bffe880: 00000000 00000000  00000000 0000000d  ................\\n\\ngoroutine 1 gp=0x14197f1321e0 m=nil [sync.WaitGroup.Wait]:\\nruntime.gopark(0x14197fe86c00?, 0x2022a4d?, 0x10?, 0x63?, 0x14197fe86c30?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197ff32be8 sp=0x14197ff32bc8 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x14197fc98610, 0x0, 0x1, 0x0, 0x19)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197ff32c50 sp=0x14197ff32be8 pc=0x1f162b2\\nsync.runtime_SemacquireWaitGroup(0x14197fd289c0?, 0xe0?)\\n\\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x14197ff32c88 sp=0x14197ff32c50 pc=0x1f3a66e\\nsync.(*WaitGroup).Wait(0x14197fc98608)\\n\\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x14197ff32cb0 sp=0x14197ff32c88 pc=0x1f4f445\\ngolang.org/x/sync/errgroup.(*Group).Wait(0x14197fc98600)\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x14197ff32cd0 sp=0x14197ff32cb0 pc=0x362507e\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x14197f521dc0, {0x59f0b08, 0x14197f530fc0}, {0x14197fcca000, 0x15, 0x1c})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x14197ff32d90 sp=0x14197ff32cd0 pc=0x3927e05\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x14197f521dc0, {0x59f0b08, 0x14197f530fc0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:500 +0x4b08 fp=0x14197ff33968 sp=0x14197ff32d90 pc=0x39261c8\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedQueryOptions).RunQuery(0x14197f1401e0, {0x59f0b08, 0x14197f530fc0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_cmd.go:75 +0x565 fp=0x14197ff33b30 sp=0x14197ff33968 pc=0x3937005\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawQueryOptions).Run(0x0?, {0x59f0b08, 0x14197f530fc0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_options.go:178 +0x54 fp=0x14197ff33b58 sp=0x14197ff33b30 pc=0x3939d34\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryCommand.func1(0x14197f4ba008?, {0x14197f5f82a0?, 0x4?, 0x553b0e2?})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_cmd.go:45 +0x28 fp=0x14197ff33b80 sp=0x14197ff33b58 pc=0x3936a68\\ngithub.com/spf13/cobra.(*Command).execute(0x14197f4ba008, {0x14197f2c0030, 0xe, 0xf})\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x14197ff33d28 sp=0x14197ff33b80 pc=0x212e614\\ngithub.com/spf13/cobra.(*Command).ExecuteC(0x14197f625b08)\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x14197ff33e18 sp=0x14197ff33d28 pc=0x212efa5\\ngithub.com/spf13/cobra.(*Command).Execute(...)\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071\\ngithub.com/spf13/cobra.(*Command).ExecuteContext(...)\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064\\nmain.main()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x14197ff33f48 sp=0x14197ff33e18 pc=0x47ddd6a\\nruntime.main()\\n\\t/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x14197ff33fe0 sp=0x14197ff33f48 pc=0x1f01975\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197ff33fe8 sp=0x14197ff33fe0 pc=0x1f40801\\n\\ngoroutine 2 gp=0x14197f132d20 m=nil [force gc (idle)]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f2fa8 sp=0x14197f1f2f88 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.forcegchelper()\\n\\t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x14197f1f2fe0 sp=0x14197f1f2fa8 pc=0x1f01c93\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f2fe8 sp=0x14197f1f2fe0 pc=0x1f40801\\ncreated by runtime.init.7 in goroutine 1\\n\\t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\\n\\ngoroutine 18 gp=0x14197f2843c0 m=nil [GC sweep wait]:\\nruntime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1ee788 sp=0x14197f1ee768 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.bgsweep(0x14197f292000)\\n\\t/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x14197f1ee7c8 sp=0x14197f1ee788 pc=0x1ee9f11\\nruntime.gcenable.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x14197f1ee7e0 sp=0x14197f1ee7c8 pc=0x1edadd7\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1ee7e8 sp=0x14197f1ee7e0 pc=0x1f40801\\ncreated by runtime.gcenable in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\\n\\ngoroutine 19 gp=0x14197f2845a0 m=nil [GC scavenge wait]:\\nruntime.gopark(0x138d6d?, 0x3979fd?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1eef78 sp=0x14197f1eef58 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.(*scavengerState).park(0x82979c0)\\n\\t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x14197f1eefa8 sp=0x14197f1eef78 pc=0x1ee77e9\\nruntime.bgscavenge(0x14197f292000)\\n\\t/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x14197f1eefc8 sp=0x14197f1eefa8 pc=0x1ee7d99\\nruntime.gcenable.gowrap2()\\n\\t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x14197f1eefe0 sp=0x14197f1eefc8 pc=0x1edad97\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1eefe8 sp=0x14197f1eefe0 pc=0x1f40801\\ncreated by runtime.gcenable in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\\n\\ngoroutine 20 gp=0x14197f284780 m=nil [GOMAXPROCS updater (idle)]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1ef788 sp=0x14197f1ef768 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.updateMaxProcsGoroutine()\\n\\t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x14197f1ef7e0 sp=0x14197f1ef788 pc=0x1f10267\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1ef7e8 sp=0x14197f1ef7e0 pc=0x1f40801\\ncreated by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\\n\\t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\\n\\ngoroutine 21 gp=0x14197f284d20 m=nil [finalizer wait]:\\nruntime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x14197f1f2601?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f2620 sp=0x14197f1f2600 pc=0x1f38d2e\\nruntime.runFinalizers()\\n\\t/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x14197f1f27e0 sp=0x14197f1f2620 pc=0x1ed9cc7\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f27e8 sp=0x14197f1f27e0 pc=0x1f40801\\ncreated by runtime.createfing in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d\\n\\ngoroutine 22 gp=0x14197f47c960 m=nil [cleanup wait]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1eff68 sp=0x14197f1eff48 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.(*cleanupQueue).dequeue(0x82980e0)\\n\\t/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x14197f1effa0 sp=0x14197f1eff68 pc=0x1ed6074\\nruntime.runCleanups()\\n\\t/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x14197f1effe0 sp=0x14197f1effa0 pc=0x1ed66e5\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1effe8 sp=0x14197f1effe0 pc=0x1f40801\\ncreated by runtime.(*cleanupQueue).createGs in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5\\n\\ngoroutine 23 gp=0x14197f5a30e0 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125b862b4ec7?, 0x1?, 0x21?, 0xab?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f0740 sp=0x14197f1f0720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x14197f29d9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f07c8 sp=0x14197f1f0740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f07e0 sp=0x14197f1f07c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f07e8 sp=0x14197f1f07e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 3 gp=0x14197f133680 m=nil [runnable]:\\nruntime.gopark(0x125b8625d223?, 0x3?, 0x89?, 0x19?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f3740 sp=0x14197f1f3720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x14197f29d9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f37c8 sp=0x14197f1f3740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f37e0 sp=0x14197f1f37c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f37e8 sp=0x14197f1f37e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 34 gp=0x14197f604000 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125b862e8e02?, 0x3?, 0xe6?, 0x2d?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60a740 sp=0x14197f60a720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x14197f29d9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f60a7c8 sp=0x14197f60a740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f60a7e0 sp=0x14197f60a7c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60a7e8 sp=0x14197f60a7e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 24 gp=0x14197f5a32c0 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125b862118da?, 0x1?, 0x4d?, 0x9a?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f0f40 sp=0x14197f1f0f20 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x14197f29d9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f0fc8 sp=0x14197f1f0f40 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f0fe0 sp=0x14197f1f0fc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f0fe8 sp=0x14197f1f0fe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 4 gp=0x14197f133860 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125b7fe90140?, 0x3?, 0x20?, 0x3a?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f3f40 sp=0x14197f1f3f20 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x14197f29d9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f3fc8 sp=0x14197f1f3f40 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f3fe0 sp=0x14197f1f3fc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f3fe8 sp=0x14197f1f3fe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 35 gp=0x14197f6041e0 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125b862a34bd?, 0x3?, 0x4?, 0x15?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60af40 sp=0x14197f60af20 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x14197f29d9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f60afc8 sp=0x14197f60af40 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f60afe0 sp=0x14197f60afc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60afe8 sp=0x14197f60afe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 25 gp=0x14197f5a34a0 m=3 mp=0x14197f1a5808 [GC mark termination]:\\nruntime.systemstack_switch()\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x14197f1f16a0 sp=0x14197f1f1690 pc=0x1f3ebc8\\nruntime.forEachP(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:2112\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x14197f1f1740 sp=0x14197f1f16a0 pc=0x1edbc34\\nruntime.gcBgMarkWorker(0x14197f29d9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x14197f1f17c8 sp=0x14197f1f1740 pc=0x1edda88\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f17e0 sp=0x14197f1f17c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f17e8 sp=0x14197f1f17e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 5 gp=0x14197f133a40 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125b8624ea9b?, 0x1?, 0xfd?, 0x7a?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f4740 sp=0x14197f1f4720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x14197f29d9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f47c8 sp=0x14197f1f4740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f47e0 sp=0x14197f1f47c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f47e8 sp=0x14197f1f47e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 50 gp=0x14197f768f00 m=nil [select, locked to thread]:\\nruntime.gopark(0x14197f60c7a8?, 0x2?, 0x1?, 0x8f?, 0x14197f60c794?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60c628 sp=0x14197f60c608 pc=0x1f38d2e\\nruntime.selectgo(0x14197f60c7a8, 0x14197f60c790, 0x0?, 0x0, 0x0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f60c758 sp=0x14197f60c628 pc=0x1f152a5\\nruntime.ensureSigM.func1()\\n\\t/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x14197f60c7e0 sp=0x14197f60c758 pc=0x1f328e8\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60c7e8 sp=0x14197f60c7e0 pc=0x1f40801\\ncreated by runtime.ensureSigM in goroutine 1\\n\\t/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5\\n\\ngoroutine 26 gp=0x14197f3b34a0 m=7 mp=0x14197f55b008 [syscall]:\\nruntime.notetsleepg(0x82bdea0, 0xffffffffffffffff)\\n\\t/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x14197f6077a0 sp=0x14197f607778 pc=0x1ece509\\nos/signal.signal_recv()\\n\\t/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x14197f6077c0 sp=0x14197f6077a0 pc=0x1f3b298\\nos/signal.loop()\\n\\t/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x14197f6077e0 sp=0x14197f6077c0 pc=0x2030553\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f6077e8 sp=0x14197f6077e0 pc=0x1f40801\\ncreated by os/signal.Notify.func1.1 in goroutine 1\\n\\t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\\n\\ngoroutine 27 gp=0x14197f6894a0 m=nil [select]:\\nruntime.gopark(0x14197f607f98?, 0x2?, 0x68?, 0x81?, 0x14197f607f8c?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f607e28 sp=0x14197f607e08 pc=0x1f38d2e\\nruntime.selectgo(0x14197f607f98, 0x14197f607f88, 0x0?, 0x0, 0x0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f607f58 sp=0x14197f607e28 pc=0x1f152a5\\nos/signal.NotifyContext.func1()\\n\\t/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x14197f607fe0 sp=0x14197f607f58 pc=0x202ff9c\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f607fe8 sp=0x14197f607fe0 pc=0x1f40801\\ncreated by os/signal.NotifyContext in goroutine 1\\n\\t/usr/lib/golang/src/os/signal/signal.go:291 +0x158\\n\\ngoroutine 224 gp=0x14197f689680 m=nil [semacquire]:\\nruntime.gopark(0x7f80904adba8?, 0x1eed7e0?, 0xc0?, 0x68?, 0x4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419800b5b58 sp=0x1419800b5b38 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x1419800b5bc0 sp=0x1419800b5b58 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x1419800b5c60 sp=0x1419800b5bc0 pc=0x1edbb12\\nruntime.gcAssistAlloc(0x14197f689680)\\n\\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x1419800b5cf0 sp=0x1419800b5c60 pc=0x1ee06b6\\nruntime.deductAssistCredit(0x7ffce51f9080?)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x1419800b5d10 sp=0x1419800b5cf0 pc=0x1ed10a6\\nruntime.mallocgc(0x7fe00, 0x4af32a0, 0x1)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x1419800b5d40 sp=0x1419800b5d10 pc=0x1f35e07\\nruntime.makeslice(0x14197fc9d0b0?, 0x141983a1fe00?, 0x20000?)\\n\\t/usr/lib/golang/src/runtime/slice.go:117 +0x49 fp=0x1419800b5d68 sp=0x1419800b5d40 pc=0x1f3b489\\nencoding/json.(*Decoder).refill(0x14197f658c80)\\n\\t/usr/lib/golang/src/encoding/json/stream.go:161 +0xee fp=0x1419800b5db8 sp=0x1419800b5d68 pc=0x200d48e\\nencoding/json.(*Decoder).readValue(0x14197f658c80)\\n\\t/usr/lib/golang/src/encoding/json/stream.go:142 +0x85 fp=0x1419800b5e08 sp=0x1419800b5db8 pc=0x200d125\\nencoding/json.(*Decoder).Decode(0x14197f658c80, {0x4cfb340, 0x14197ffa44c0})\\n\\t/usr/lib/golang/src/encoding/json/stream.go:65 +0x75 fp=0x1419800b5e38 sp=0x1419800b5e08 pc=0x200cdf5\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readPrimaryTable(0x14197f8c54f0, 0x14197fb50000)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:264 +0x16e fp=0x1419800b5ec8 sp=0x1419800b5e38 pc=0x38d822e\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x14197f8c54f0)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:174 +0x189 fp=0x1419800b5f30 sp=0x1419800b5ec8 pc=0x38d7b89\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x14197f8c54f0, 0x14198007be90)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x1419800b5fc0 sp=0x1419800b5f30 pc=0x38d788c\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x1419800b5fe0 sp=0x1419800b5fc0 pc=0x38d751b\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1419800b5fe8 sp=0x1419800b5fe0 pc=0x1f40801\\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 183\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc\\n\\ngoroutine 87 gp=0x14197fd3c000 m=nil [sync.WaitGroup.Wait]:\\nruntime.gopark(0x14197fcf9e00?, 0x1ed0734?, 0x20?, 0x94?, 0x7f80917bdf06?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf9da0 sp=0x14197fcf9d80 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x14197f69cc50, 0x0, 0x1, 0x0, 0x19)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fcf9e08 sp=0x14197fcf9da0 pc=0x1f162b2\\nsync.runtime_SemacquireWaitGroup(0x14197f86b2d8?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x14197fcf9e40 sp=0x14197fcf9e08 pc=0x1f3a66e\\nsync.(*WaitGroup).Wait(0x14197f69cc48)\\n\\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x14197fcf9e68 sp=0x14197fcf9e40 pc=0x1f4f445\\ngolang.org/x/sync/errgroup.(*Group).Wait(0x14197f69cc40)\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x14197fcf9e88 sp=0x14197fcf9e68 pc=0x362507e\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x14197f521d80, {0x59f0b40, 0x14197f8c4b40}, {0x14197fcca000, 0x15, 0x6f63746669687372?}, 0x14197f1c02a0)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x14197fcf9f00 sp=0x14197fcf9e88 pc=0x392976a\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x14197fcf9f78 sp=0x14197fcf9f00 pc=0x392808e\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197fcf9fe0 sp=0x14197fcf9f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf9fe8 sp=0x14197fcf9fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 77 gp=0x14197fd3c1e0 m=nil [IO wait]:\\nruntime.gopark(0x14197fb3e4a0?, 0x1edb176?, 0x80?, 0xed?, 0xb?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f202808 sp=0x14197f2027e8 pc=0x1f38d2e\\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x14197f202840 sp=0x14197f202808 pc=0x1ef9e57\\ninternal/poll.runtime_pollWait(0x7f8098109000, 0x72)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x14197f202860 sp=0x14197f202840 pc=0x1f375c5\\ninternal/poll.(*pollDesc).wait(0x14197f323a80?, 0x14197f34d300?, 0x0)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x14197f202888 sp=0x14197f202860 pc=0x1fbd167\\ninternal/poll.(*pollDesc).waitRead(...)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\\ninternal/poll.(*FD).Read(0x14197f323a80, {0x14197f34d300, 0x1300, 0x1300})\\n\\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x14197f202920 sp=0x14197f202888 pc=0x1fbe4ce\\nnet.(*netFD).Read(0x14197f323a80, {0x14197f34d300?, 0x14197f34d30a?, 0x5?})\\n\\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x14197f202968 sp=0x14197f202920 pc=0x20798c5\\nnet.(*conn).Read(0x14197f1300d8, {0x14197f34d300?, 0x7f809057a868?, 0x7f80df53a6d8?})\\n\\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x14197f2029b0 sp=0x14197f202968 pc=0x208b7a5\\ncrypto/tls.(*atLeastReader).Read(0x14197f821a10, {0x14197f34d300?, 0x14197f202a98?, 0x1efcc28?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x14197f2029f8 sp=0x14197f2029b0 pc=0x2222ddb\\nbytes.(*Buffer).ReadFrom(0x14197f8a57a8, {0x59a3680, 0x14197f821a10})\\n\\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x14197f202a50 sp=0x14197f2029f8 pc=0x1fe9258\\ncrypto/tls.(*Conn).readFromUntil(0x14197f8a5508, {0x59a32c0, 0x14197f1300d8}, 0x222764c?)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x14197f202a88 sp=0x14197f202a50 pc=0x2222fbe\\ncrypto/tls.(*Conn).readRecordOrCCS(0x14197f8a5508, 0x0)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x14197f202c68 sp=0x14197f202a88 pc=0x222009b\\ncrypto/tls.(*Conn).readRecord(...)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:588\\ncrypto/tls.(*Conn).Read(0x14197f8a5508, {0x14197fa26000, 0x1000, 0x4e55540?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x14197f202cd0 sp=0x14197f202c68 pc=0x22267c5\\nnet/http.(*persistConn).Read(0x14197f8032c0, {0x14197fa26000?, 0x59a0ea0?, 0x81260e0?})\\n\\t/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x14197f202d30 sp=0x14197f202cd0 pc=0x2319d07\\nbufio.(*Reader).fill(0x14197f29b620)\\n\\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f202d68 sp=0x14197f202d30 pc=0x20a6c63\\nbufio.(*Reader).Peek(0x14197f29b620, 0x1)\\n\\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x14197f202d80 sp=0x14197f202d68 pc=0x20a6df3\\nnet/http.(*persistConn).readLoop(0x14197f8032c0)\\n\\t/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x14197f202fc8 sp=0x14197f202d80 pc=0x231a812\\nnet/http.(*Transport).dialConn.gowrap2()\\n\\t/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x14197f202fe0 sp=0x14197f202fc8 pc=0x2319197\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f202fe8 sp=0x14197f202fe0 pc=0x1f40801\\ncreated by net/http.(*Transport).dialConn in goroutine 9\\n\\t/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac\\n\\ngoroutine 7 gp=0x14197fe025a0 m=nil [sync.Mutex.Lock]:\\nruntime.gopark(0x14197f9d3c68?, 0x14197f9d3c70?, 0x94?, 0x69?, 0x14197f9d3c78?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f9d3c00 sp=0x14197f9d3be0 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x14197fccaad4, 0x0, 0x3, 0x2, 0x16)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197f9d3c68 sp=0x14197f9d3c00 pc=0x1f162b2\\ninternal/sync.runtime_SemacquireMutex(0x1000?, 0xd0?, 0x1ed11a5?)\\n\\t/usr/lib/golang/src/runtime/sema.go:95 +0x25 fp=0x14197f9d3ca0 sp=0x14197f9d3c68 pc=0x1f3a545\\ninternal/sync.(*Mutex).lockSlow(0x14197fccaad0)\\n\\t/usr/lib/golang/src/internal/sync/mutex.go:149 +0x15d fp=0x14197f9d3cf0 sp=0x14197f9d3ca0 pc=0x1f4d31d\\ninternal/sync.(*Mutex).Lock(...)\\n\\t/usr/lib/golang/src/internal/sync/mutex.go:70\\nsync.(*Mutex).Lock(...)\\n\\t/usr/lib/golang/src/sync/mutex.go:46\\nnet/http.(*http2clientConnReadLoop).streamByID(0x22babb5?, 0x1b, 0x1)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:10083 +0x5b fp=0x14197f9d3d48 sp=0x14197f9d3cf0 pc=0x22e10db\\nnet/http.(*http2clientConnReadLoop).processData(0x14197f9d3fa8, 0x14197fc9d200)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9939 +0x45 fp=0x14197f9d3e18 sp=0x14197f9d3d48 pc=0x22e05e5\\nnet/http.(*http2clientConnReadLoop).run(0x14197f9d3fa8)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9584 +0x4b6 fp=0x14197f9d3f60 sp=0x14197f9d3e18 pc=0x22de536\\nnet/http.(*http2ClientConn).readLoop(0x14197fccaa80)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x14197f9d3fc8 sp=0x14197f9d3f60 pc=0x22dd532\\nnet/http.(*http2Transport).newClientConn.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x14197f9d3fe0 sp=0x14197f9d3fc8 pc=0x22d7057\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f9d3fe8 sp=0x14197f9d3fe0 pc=0x1f40801\\ncreated by net/http.(*http2Transport).newClientConn in goroutine 6\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5\\n\\ngoroutine 78 gp=0x14197fe02960 m=nil [select]:\\nruntime.gopark(0x14197f9f7f48?, 0x2?, 0x0?, 0xb6?, 0x14197f9f7ef4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf2d88 sp=0x14197fcf2d68 pc=0x1f38d2e\\nruntime.selectgo(0x14197fcf2f48, 0x14197f9f7ef0, 0x14197f1cd0c0?, 0x0, 0x14197f89c2a0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197fcf2eb8 sp=0x14197fcf2d88 pc=0x1f152a5\\nnet/http.(*persistConn).writeLoop(0x14197f8032c0)\\n\\t/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x14197fcf2fc8 sp=0x14197fcf2eb8 pc=0x231c426\\nnet/http.(*Transport).dialConn.gowrap3()\\n\\t/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x14197fcf2fe0 sp=0x14197fcf2fc8 pc=0x2319157\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf2fe8 sp=0x14197fcf2fe0 pc=0x1f40801\\ncreated by net/http.(*Transport).dialConn in goroutine 9\\n\\t/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe\\n\\ngoroutine 163 gp=0x14197f7694a0 m=nil [select]:\\nruntime.gopark(0x14197fdd3f08?, 0x6?, 0xa0?, 0x94?, 0x14197fdd3da4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f205c08 sp=0x14197f205be8 pc=0x1f38d2e\\nruntime.selectgo(0x14197f205f08, 0x14197fdd3d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f205d38 sp=0x14197f205c08 pc=0x1f152a5\\nnet/http.(*http2clientStream).writeRequest(0x14197f37f380, 0x14197f49e140, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x14197f205f90 sp=0x14197f205d38 pc=0x22da6a8\\nnet/http.(*http2clientStream).doRequest(0x14197f37f380, 0x14197f546100?, 0x14197f609798?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x14197f205fb8 sp=0x14197f205f90 pc=0x22d99d8\\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197f205fe0 sp=0x14197f205fb8 pc=0x22d999f\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f205fe8 sp=0x14197f205fe0 pc=0x1f40801\\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 184\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\\n\\ngoroutine 88 gp=0x14197f769a40 m=nil [semacquire]:\\nruntime.gopark(0x141980119958?, 0x1ed3e05?, 0xe8?, 0xd2?, 0x7f80904ad2e8?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x141980119908 sp=0x1419801198e8 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x141980119970 sp=0x141980119908 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x141980119a10 sp=0x141980119970 pc=0x1edbb12\\nruntime.gcAssistAlloc(0x14197f769a40)\\n\\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x141980119aa0 sp=0x141980119a10 pc=0x1ee06b6\\nruntime.deductAssistCredit(0x141983b00000?)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x141980119ac0 sp=0x141980119aa0 pc=0x1ed10a6\\nruntime.mallocgc(0x7edd, 0x0, 0x0)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x141980119af0 sp=0x141980119ac0 pc=0x1f35e07\\nruntime.slicebytetostring(0x4e537a0?, 0x141983b00000, 0x7edd)\\n\\t/usr/lib/golang/src/runtime/string.go:170 +0x77 fp=0x141980119b20 sp=0x141980119af0 pc=0x1f1e397\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x14197f8c4b40}, 0x14197f1c03f0, 0x14197f531740)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:322 +0x91b fp=0x141980119f48 sp=0x141980119b20 pc=0x3920bfb\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x141980119f78 sp=0x141980119f48 pc=0x392800c\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x141980119fe0 sp=0x141980119f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x141980119fe8 sp=0x141980119fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 110 gp=0x14197fd3cb40 m=nil [sync.Mutex.Lock]:\\nruntime.gopark(0x22b8a73?, 0x14197f7e1000?, 0xb0?, 0x4c?, 0x8160a40?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1c7510 sp=0x14197f1c74f0 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x14197fccaad4, 0x0, 0x3, 0x2, 0x16)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197f1c7578 sp=0x14197f1c7510 pc=0x1f162b2\\ninternal/sync.runtime_SemacquireMutex(0x14197f1c75d0?, 0x66?, 0x61f?)\\n\\t/usr/lib/golang/src/runtime/sema.go:95 +0x25 fp=0x14197f1c75b0 sp=0x14197f1c7578 pc=0x1f3a545\\ninternal/sync.(*Mutex).lockSlow(0x14197fccaad0)\\n\\t/usr/lib/golang/src/internal/sync/mutex.go:149 +0x15d fp=0x14197f1c7600 sp=0x14197f1c75b0 pc=0x1f4d31d\\ninternal/sync.(*Mutex).Lock(...)\\n\\t/usr/lib/golang/src/internal/sync/mutex.go:70\\nsync.(*Mutex).Lock(...)\\n\\t/usr/lib/golang/src/sync/mutex.go:46\\nnet/http.http2transportResponseBody.Read({0x1f3ec12?}, {0x14198313d000?, 0x1eed797?, 0x14197f1c76b0?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9875 +0x209 fp=0x14197f1c7688 sp=0x14197f1c7600 pc=0x22dff89\\nbufio.(*Reader).fill(0x14197f3a00c0)\\n\\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f1c76c0 sp=0x14197f1c7688 pc=0x20a6c63\\nbufio.(*Reader).ReadByte(0x14197f3a00c0)\\n\\t/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x14197f1c76d8 sp=0x14197f1c76c0 pc=0x20a73c7\\ncompress/flate.(*decompressor).moreBits(0x141980059908)\\n\\t/usr/lib/golang/src/compress/flate/inflate.go:697 +0x24 fp=0x14197f1c7710 sp=0x14197f1c76d8 pc=0x2288824\\ncompress/flate.(*decompressor).nextBlock(0x141980059908)\\n\\t/usr/lib/golang/src/compress/flate/inflate.go:304 +0x28 fp=0x14197f1c7728 sp=0x14197f1c7710 pc=0x2286ba8\\ncompress/flate.(*decompressor).Read(0x141980059908, {0x14198314e000, 0x1000, 0x7f8090ef0f00?})\\n\\t/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x14197f1c7758 sp=0x14197f1c7728 pc=0x2286dbb\\ncompress/gzip.(*Reader).Read(0x14197f672dc8, {0x14198314e000, 0x1000, 0x1000})\\n\\t/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x14197f1c77e8 sp=0x14197f1c7758 pc=0x228c838\\ngithub.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x410d0fa76c5d7d40?, {0x14198314e000?, 0x7f8090ef0d20?, 0x52ffb40?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x14197f1c7818 sp=0x14197f1c77e8 pc=0x38b33e2\\nbufio.(*Reader).fill(0x14197f3a0120)\\n\\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f1c7850 sp=0x14197f1c7818 pc=0x20a6c63\\nbufio.(*Reader).Peek(0x14197f3a0120, 0x1)\\n\\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x14197f1c7868 sp=0x14197f1c7850 pc=0x20a6df3\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.validateJsonResponse(0x14197f3a0120)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:38 +0x25 fp=0x14197f1c78b8 sp=0x14197f1c7868 pc=0x38d6be5\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.newFrameReader({0x59db110, 0x14197fe1e040}, {0x59f0b40, 0x14197f748140})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:26 +0x188 fp=0x14197f1c7960 sp=0x14197f1c78b8 pc=0x38d6a08\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset({0x59f0bb0?, 0x14197f4797a0?}, {0x59db110, 0x14197fe1e040}, 0x1, 0x3e8, 0x1)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:66 +0x19e fp=0x14197f1c79e0 sp=0x14197f1c7960 pc=0x38d737e\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x14197f521d40, {0x59f0bb0, 0x14197f4797a0}, {0x14197f49a330, 0xb}, 0x14197fe1e700, {0x0?, 0x2?, 0x0?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:167 +0x2b7 fp=0x14197f1c7a78 sp=0x14197f1c79e0 pc=0x38e4f17\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197f7a2600}, 0x14197f1c02a0)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x14197f1c7d60 sp=0x14197f1c7a78 pc=0x391b5c5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x14197f521d80, {0x59f0b40, 0x14197fdd8140}, {0x5a00340, 0x14197fc1e5d0}, 0x14197f1c02a0)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:98 +0x366 fp=0x14197f1c7e88 sp=0x14197f1c7d60 pc=0x3929ec6\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14197f1c7f78 sp=0x14197f1c7e88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197f1c7fe0 sp=0x14197f1c7f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1c7fe8 sp=0x14197f1c7fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 89 gp=0x14197fd82960 m=nil [select]:\\nruntime.gopark(0x1419800b7d00?, 0x2?, 0xd7?, 0xfd?, 0x1419800b7c74?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419800b7ac8 sp=0x1419800b7aa8 pc=0x1f38d2e\\nruntime.selectgo(0x1419800b7d00, 0x1419800b7c70, 0x1419801521c0?, 0x0, 0x14197f5daba0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x1419800b7bf8 sp=0x1419800b7ac8 pc=0x1f152a5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x14197f521d80?, {0x59f0b40, 0x14197f8c4b40}, 0x14197f1c02a0, 0x14197f1c03f0)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x1419800b7f10 sp=0x1419800b7bf8 pc=0x392836e\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x1419800b7f78 sp=0x1419800b7f10 pc=0x3927f3d\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x1419800b7fe0 sp=0x1419800b7f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1419800b7fe8 sp=0x1419800b7fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 175 gp=0x14197fd3d2c0 m=nil [sync.Cond.Wait]:\\nruntime.gopark(0x22c2d5b?, 0x14197fe1c080?, 0x0?, 0xe0?, 0x14197f777000?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14198012b978 sp=0x14198012b958 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nsync.runtime_notifyListWait(0x14197f37f3c8, 0x3c)\\n\\t/usr/lib/golang/src/runtime/sema.go:617 +0x1b3 fp=0x14198012b9c8 sp=0x14198012b978 pc=0x1f3a8f3\\nsync.(*Cond).Wait(0x1000?)\\n\\t/usr/lib/golang/src/sync/cond.go:71 +0x73 fp=0x14198012ba00 sp=0x14198012b9c8 pc=0x1f4d673\\nnet/http.(*http2pipe).Read(0x14197f37f3b0, {0x14197ff7e000, 0x1000, 0x1000})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:3928 +0xdb fp=0x14198012ba68 sp=0x14198012ba00 pc=0x22c2c1b\\nnet/http.http2transportResponseBody.Read({0x4a54?}, {0x14197ff7e000?, 0x14198012bb80?, 0x2287949?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9852 +0x59 fp=0x14198012baf0 sp=0x14198012ba68 pc=0x22dfdd9\\nbufio.(*Reader).fill(0x14197fcaa000)\\n\\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14198012bb28 sp=0x14198012baf0 pc=0x20a6c63\\nbufio.(*Reader).ReadByte(0x14197fcaa000)\\n\\t/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x14198012bb40 sp=0x14198012bb28 pc=0x20a73c7\\ncompress/flate.(*decompressor).moreBits(0x141980056008)\\n\\t/usr/lib/golang/src/compress/flate/inflate.go:697 +0x24 fp=0x14198012bb78 sp=0x14198012bb40 pc=0x2288824\\ncompress/flate.(*decompressor).nextBlock(0x141980056008)\\n\\t/usr/lib/golang/src/compress/flate/inflate.go:304 +0x28 fp=0x14198012bb90 sp=0x14198012bb78 pc=0x2286ba8\\ncompress/flate.(*decompressor).Read(0x141980056008, {0x141980034b13, 0x4ed, 0xb13?})\\n\\t/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x14198012bbc0 sp=0x14198012bb90 pc=0x2286dbb\\ncompress/gzip.(*Reader).Read(0x141980000b08, {0x141980034b13, 0x4ed, 0x4ed})\\n\\t/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x14198012bc50 sp=0x14198012bbc0 pc=0x228c838\\ngithub.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x14198012bcf0?, {0x141980034b13?, 0x1000?, 0x0?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x14198012bc80 sp=0x14198012bc50 pc=0x38b33e2\\nbufio.(*Reader).fill(0x141980807010)\\n\\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14198012bcb8 sp=0x14198012bc80 pc=0x20a6c63\\nbufio.(*Reader).ReadSlice(0x141980807010, 0xa)\\n\\t/usr/lib/golang/src/bufio/bufio.go:377 +0x29 fp=0x14198012bd00 sp=0x14198012bcb8 pc=0x20a7769\\nbufio.(*Reader).collectFragments(0x141980807010, 0xa)\\n\\t/usr/lib/golang/src/bufio/bufio.go:452 +0x85 fp=0x14198012bde0 sp=0x14198012bd00 pc=0x20a7b85\\nbufio.(*Reader).ReadBytes(0x4bbc5a0?, 0xa0?)\\n\\t/usr/lib/golang/src/bufio/bufio.go:479 +0x17 fp=0x14198012be60 sp=0x14198012bde0 pc=0x20a7e77\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.(*frameReader).advance(0x141980807000)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:64 +0x45 fp=0x14198012beb0 sp=0x14198012be60 pc=0x38d6d45\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readRoutine(0x141980807000, 0x14197f8c8370)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:85 +0x51 fp=0x14198012bfc0 sp=0x14198012beb0 pc=0x38d7591\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap2()\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x1b fp=0x14198012bfe0 sp=0x14198012bfc0 pc=0x38d74db\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14198012bfe8 sp=0x14198012bfe0 pc=0x1f40801\\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 184\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x24a\\n\\ngoroutine 183 gp=0x14197fd3d860 m=nil [semacquire]:\\nruntime.gopark(0x7f8090f36c08?, 0x1eed7e0?, 0x80?, 0x43?, 0x2?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fac5208 sp=0x14197fac51e8 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fac5270 sp=0x14197fac5208 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x14197fac5310 sp=0x14197fac5270 pc=0x1edbb12\\nruntime.gcAssistAlloc(0x14197fd3d860)\\n\\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x14197fac53a0 sp=0x14197fac5310 pc=0x1ee06b6\\nruntime.deductAssistCredit(0x14197fac53f0?)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x14197fac53c0 sp=0x14197fac53a0 pc=0x1ed10a6\\nruntime.mallocgc(0x5500, 0x0, 0x0)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x14197fac53f0 sp=0x14197fac53c0 pc=0x1f35e07\\nruntime.growslice(0x141983adc000, 0x14197fac54a8?, 0x1ed3be5?, 0x82bed80?, 0x1ed0417?)\\n\\t/usr/lib/golang/src/runtime/slice.go:265 +0x573 fp=0x14197fac5460 sp=0x14197fac53f0 pc=0x1f3ba93\\nstrings.(*Builder).Write(0x14197f569ea0, {0x1419839fd000, 0x1000, 0x1000000010765?})\\n\\t/usr/lib/golang/src/strings/builder.go:89 +0x90 fp=0x14197fac54b0 sp=0x14197fac5460 pc=0x1ff0cb0\\nbufio.(*Writer).Flush(0x14197fac5698)\\n\\t/usr/lib/golang/src/bufio/bufio.go:642 +0x52 fp=0x14197fac54f8 sp=0x14197fac54b0 pc=0x20a8832\\nbufio.(*Writer).WriteString(0x14197fac5698, {0x553a4f6?, 0x553a9e6?})\\n\\t/usr/lib/golang/src/bufio/bufio.go:766 +0x165 fp=0x14197fac5568 sp=0x14197fac54f8 pc=0x20a8f65\\nencoding/csv.(*Writer).Write(0x14197fac5688, {0x14198384d480?, 0x4, 0x141983909500?})\\n\\t/usr/lib/golang/src/encoding/csv/writer.go:92 +0x2b4 fp=0x14197fac55d0 sp=0x14197fac5568 pc=0x20ad3d4\\ngithub.com/Azure/azure-kusto-go/azkustodata/query.(*row).String(0x5e?)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/row_impl.go:101 +0x21f fp=0x14197fac56e8 sp=0x14197fac55d0 pc=0x38cf67f\\nfmt.(*pp).handleMethods(0x14197fc20000, 0x7fac58f0?)\\n\\t/usr/lib/golang/src/fmt/print.go:674 +0x2f4 fp=0x14197fac58a8 sp=0x14197fac56e8 pc=0x1fdd1d4\\nfmt.(*pp).printArg(0x14197fc20000, {0x549a3a0, 0x14197fd92e80}, 0x76)\\n\\t/usr/lib/golang/src/fmt/print.go:757 +0x42e fp=0x14197fac5938 sp=0x14197fac58a8 pc=0x1fdd9ee\\nfmt.(*pp).doPrintf(0x14197fc20000, {0x553a400, 0x2}, {0x14197fac5d08, 0x1, 0x1})\\n\\t/usr/lib/golang/src/fmt/print.go:1075 +0x3ec fp=0x14197fac5a20 sp=0x14197fac5938 pc=0x1fe04ec\\nfmt.Sprintf({0x553a400, 0x2}, {0x14197fac5d08, 0x1, 0x1})\\n\\t/usr/lib/golang/src/fmt/print.go:239 +0x53 fp=0x14197fac5a78 sp=0x14197fac5a20 pc=0x1fda6b3\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197fc4b200}, 0x14197f1c02a0)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:160 +0x8d8 fp=0x14197fac5d60 sp=0x14197fac5a78 pc=0x391bad8\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x14197fdd8140?}, {0x5a00300?, 0x14197fc4b200?}, 0x0?)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x14197fac5e88 sp=0x14197fac5d60 pc=0x3929c16\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14197fac5f78 sp=0x14197fac5e88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197fac5fe0 sp=0x14197fac5f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fac5fe8 sp=0x14197fac5fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 184 gp=0x14197fd3da40 m=nil [chan receive]:\\nruntime.gopark(0x14197f3c29a0?, 0x2034990?, 0x10?, 0xea?, 0x59f0ad0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14198011f9d8 sp=0x14198011f9b8 pc=0x1f38d2e\\nruntime.chanrecv(0x14197f3fa700, 0x14198011fcb8, 0x1)\\n\\t/usr/lib/golang/src/runtime/chan.go:667 +0x4ae fp=0x14198011fa50 sp=0x14198011f9d8 pc=0x1ec7d2e\\nruntime.chanrecv2(0x5a09ed0?, 0x14197f530f90?)\\n\\t/usr/lib/golang/src/runtime/chan.go:514 +0x12 fp=0x14198011fa78 sp=0x14198011fa50 pc=0x1ec7872\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197fc4b340}, 0x14197f1c02a0)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:141 +0x94d fp=0x14198011fd60 sp=0x14198011fa78 pc=0x391bb4d\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x74666968736e6570?, {0x59f0b40?, 0x14197fdd8140?}, {0x5a00300?, 0x14197fc4b340?}, 0x6d736f63222c2273?)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x14198011fe88 sp=0x14198011fd60 pc=0x3929c16\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14198011ff78 sp=0x14198011fe88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14198011ffe0 sp=0x14198011ff78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14198011ffe8 sp=0x14198011ffe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 135 gp=0x14197fb465a0 m=nil [semacquire]:\\nruntime.gopark(0x14197f454000?, 0x22dc6c0?, 0xa0?, 0x82?, 0x1419808322a0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fe99ce8 sp=0x14197fe99cc8 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fe99d50 sp=0x14197fe99ce8 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x14197fe99df0 sp=0x14197fe99d50 pc=0x1edbb12\\nruntime.gcAssistAlloc(0x14197fb465a0)\\n\\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x14197fe99e80 sp=0x14197fe99df0 pc=0x1ee06b6\\nruntime.deductAssistCredit(0x0?)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x14197fe99ea0 sp=0x14197fe99e80 pc=0x1ed10a6\\nruntime.mallocgc(0x70, 0x0, 0x1)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x14197fe99ed0 sp=0x14197fe99ea0 pc=0x1f35e07\\nruntime.makechan(0x0?, 0x0)\\n\\t/usr/lib/golang/src/runtime/chan.go:99 +0x126 fp=0x14197fe99f10 sp=0x14197fe99ed0 pc=0x1ec6826\\nnet/http.(*http2clientStream).cleanupWriteRequest(0x14197f2f1200, {0x0, 0x0})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8951 +0xd6 fp=0x14197fe99f90 sp=0x14197fe99f10 pc=0x22db1b6\\nnet/http.(*http2clientStream).doRequest(0x14197f2f1200, 0x14197f7480a0?, 0x14197f4d0020?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8719 +0x28 fp=0x14197fe99fb8 sp=0x14197fe99f90 pc=0x22d99e8\\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197fe99fe0 sp=0x14197fe99fb8 pc=0x22d999f\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fe99fe8 sp=0x14197fe99fe0 pc=0x1f40801\\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 110\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\\n\\ngoroutine 90 gp=0x14197fd82b40 m=nil [select]:\\nruntime.gopark(0x14197ff1df08?, 0x6?, 0x40?, 0x2b?, 0x14197ff1dda4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f203c08 sp=0x14197f203be8 pc=0x1f38d2e\\nruntime.selectgo(0x14197f203f08, 0x14197ff1dd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f203d38 sp=0x14197f203c08 pc=0x1f152a5\\nnet/http.(*http2clientStream).writeRequest(0x14197f2f0f00, 0x14197fb50140, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x14197f203f90 sp=0x14197f203d38 pc=0x22da6a8\\nnet/http.(*http2clientStream).doRequest(0x14197f2f0f00, 0x0?, 0x14197f1f1f98?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x14197f203fb8 sp=0x14197f203f90 pc=0x22d99d8\\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197f203fe0 sp=0x14197f203fb8 pc=0x22d999f\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f203fe8 sp=0x14197f203fe0 pc=0x1f40801\\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 183\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\\n\\ngoroutine 225 gp=0x14197fd834a0 m=nil [sync.Cond.Wait]:\\nruntime.gopark(0x22b8a73?, 0x141980390000?, 0x0?, 0x60?, 0xe?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419801298f8 sp=0x1419801298d8 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nsync.runtime_notifyListWait(0x14197f2f0f48, 0x3c)\\n\\t/usr/lib/golang/src/runtime/sema.go:617 +0x1b3 fp=0x141980129948 sp=0x1419801298f8 pc=0x1f3a8f3\\nsync.(*Cond).Wait(0x14197fbb1640?)\\n\\t/usr/lib/golang/src/sync/cond.go:71 +0x73 fp=0x141980129980 sp=0x141980129948 pc=0x1f4d673\\nnet/http.(*http2pipe).Read(0x14197f2f0f30, {0x14197fd98000, 0x1000, 0x1000})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:3928 +0xdb fp=0x1419801299e8 sp=0x141980129980 pc=0x22c2c1b\\nnet/http.http2transportResponseBody.Read({0x141980129a60?}, {0x14197fd98000?, 0x20a6c63?, 0x1f3ec12?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9852 +0x59 fp=0x141980129a70 sp=0x1419801299e8 pc=0x22dfdd9\\nbufio.(*Reader).fill(0x14197fc90180)\\n\\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x141980129aa8 sp=0x141980129a70 pc=0x20a6c63\\nbufio.(*Reader).ReadByte(0x14197fc90180)\\n\\t/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x141980129ac0 sp=0x141980129aa8 pc=0x20a73c7\\ncompress/flate.(*decompressor).huffSym(0x141980058608, 0x141980058638)\\n\\t/usr/lib/golang/src/compress/flate/inflate.go:720 +0x7e fp=0x141980129b10 sp=0x141980129ac0 pc=0x228893e\\ncompress/flate.(*decompressor).huffmanBlock(0x82bed80?)\\n\\t/usr/lib/golang/src/compress/flate/inflate.go:495 +0x45 fp=0x141980129b90 sp=0x141980129b10 pc=0x2287605\\ncompress/flate.(*decompressor).Read(0x141980058608, {0x14197fd99cea, 0x316, 0xcea?})\\n\\t/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x141980129bc0 sp=0x141980129b90 pc=0x2286dbb\\ncompress/gzip.(*Reader).Read(0x14197f3d8848, {0x14197fd99cea, 0x316, 0x316})\\n\\t/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x141980129c50 sp=0x141980129bc0 pc=0x228c838\\ngithub.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x141980129cf0?, {0x14197fd99cea?, 0x1000?, 0x0?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x141980129c80 sp=0x141980129c50 pc=0x38b33e2\\nbufio.(*Reader).fill(0x14197f546110)\\n\\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x141980129cb8 sp=0x141980129c80 pc=0x20a6c63\\nbufio.(*Reader).ReadSlice(0x14197f546110, 0xa)\\n\\t/usr/lib/golang/src/bufio/bufio.go:377 +0x29 fp=0x141980129d00 sp=0x141980129cb8 pc=0x20a7769\\nbufio.(*Reader).collectFragments(0x14197f546110, 0xa)\\n\\t/usr/lib/golang/src/bufio/bufio.go:452 +0x85 fp=0x141980129de0 sp=0x141980129d00 pc=0x20a7b85\\nbufio.(*Reader).ReadBytes(0x4bbc5a0?, 0xa0?)\\n\\t/usr/lib/golang/src/bufio/bufio.go:479 +0x17 fp=0x141980129e60 sp=0x141980129de0 pc=0x20a7e77\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.(*frameReader).advance(0x14197f546100)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:64 +0x45 fp=0x141980129eb0 sp=0x141980129e60 pc=0x38d6d45\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readRoutine(0x14197f546100, 0x14197f8c54f0)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:85 +0x51 fp=0x141980129fc0 sp=0x141980129eb0 pc=0x38d7591\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap2()\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x1b fp=0x141980129fe0 sp=0x141980129fc0 pc=0x38d74db\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x141980129fe8 sp=0x141980129fe0 pc=0x1f40801\\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 183\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x24a\\n\\ngoroutine 174 gp=0x14197ff64000 m=nil [select]:\\nruntime.gopark(0x1419800b9df8?, 0x2?, 0x0?, 0x0?, 0x1419800b9db4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf3c50 sp=0x14197fcf3c30 pc=0x1f38d2e\\nruntime.selectgo(0x14197fcf3df8, 0x1419800b9db0, 0x59d87c8?, 0x0, 0x14197f2980e0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197fcf3d80 sp=0x14197fcf3c50 pc=0x1f152a5\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.nextFrame(0x14197f8c8370)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:198 +0x96 fp=0x14197fcf3e38 sp=0x14197fcf3d80 pc=0x38d7d16\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readPrimaryTable(0x14197f8c8370, 0x14197fb50000)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:258 +0xb1 fp=0x14197fcf3ec8 sp=0x14197fcf3e38 pc=0x38d8171\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x14197f8c8370)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:174 +0x189 fp=0x14197fcf3f30 sp=0x14197fcf3ec8 pc=0x38d7b89\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x14197f8c8370, 0x14197f59aee0)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x14197fcf3fc0 sp=0x14197fcf3f30 pc=0x38d788c\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x14197fcf3fe0 sp=0x14197fcf3fc0 pc=0x38d751b\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf3fe8 sp=0x14197fcf3fe0 pc=0x1f40801\\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 184\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc\\n\\nrax    0x0\\nrbx    0x7f808bfff640\\nrcx    0x7f80df60594c\\nrdx    0x6\\nrdi    0x109fd\\nrsi    0x10a00\\nrbp    0x10a00\\nrsp    0x7f808bffe790\\nr8     0x7f808bffe860\\nr9     0x7f80df72b4e0\\nr10    0x8\\nr11    0x246\\nr12    0x6\\nr13    0x7f808bffeb60\\nr14    0x14197f1334a0\\nr15    0xffffffffffffffff\\nrip    0x7f80df60594c\\nrflags 0x246\\ncs     0x33\\nfs     0x0\\ngs     0x0\\n\\n  query-infra-svc: command failed: exit status 2\\noutput: runtime/cgo: pthread_create failed: Resource temporarily unavailable\\nSIGABRT: abort\\nPC=0x7f7ef9baa94c m=3 sigcode=18446744073709551610\\n\\ngoroutine 0 gp=0x170ac5c1f0e0 m=3 mp=0x170ac5c91808 [idle]:\\nruntime: g 0 gp=0x170ac5c1f0e0: unknown pc 0x7f7ef9baa94c\\nstack: frame={sp:0x7f7eabffe790, fp:0x0} stack=[0x7f7eab7ff1e0,0x7f7eabffede0)\\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\\n00007f7eabffe690: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6a0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6b0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6c0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6d0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6e0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6f0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe700: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe710: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe720: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe730: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe740: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe750: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe760: 00000000 00000000  00007f7e f9bb7fa9  ............~...\\n00007f7eabffe770: 00000000 00000000  00000000 00000011  ................\\n00007f7eabffe780: 00000000 00000130  00007f7e f9baa93e  0.......\u003e...~...\\n00007f7eabffe790: 00000000 00000120  00000000 00000000   ...............\\n                  ^ SP\\n00007f7eabffe7a0: 00000000 00000013  00000000 00000000  ................\\n00007f7eabffe7b0: 00000000 00000000  00007f7e f9c2dcff  ............~...\\n00007f7eabffe7c0: 00007f7e 7bfff910  00007f7e f9c2dcb1  ...{~.......~...\\n00007f7eabffe7d0: 00007f7e 7bfff640  00007f7e abffeaa0  @..{~.......~...\\n00007f7eabffe7e0: 00007f7e abffe90e  00007f7e abffe90f  ....~.......~...\\n00007f7eabffe7f0: 00007f7e 7bfff640  00007f7e f9ba87f5  @..{~.......~...\\n00007f7eabffe800: 00007f7e a4000020  00000000 00000120   ...~... .......\\n00007f7eabffe810: 00000000 003d0f00  176a34c4 bac87500  ..=......u...4j.\\n00007f7eabffe820: 00007f7e abfff640  00000000 00000006  @...~...........\\n00007f7eabffe830: 00007f7e a4000f30  00007f7e abffeb60  0...~...`...~...\\n00007f7eabffe840: 0000170a c5c1f0e0  00007f7e f9b5d646  ........F...~...\\n00007f7eabffe850: 00007f7e f9d1ae90  00007f7e f9b477f3  ....~....w..~...\\n00007f7eabffe860: 00000000 00000020  176a34c4 bac87500   ........u...4j.\\n00007f7eabffe870: 00007f7e 7b7ff000  00007f7e f9c1cac4  ...{~.......~...\\n00007f7eabffe880: 00000000 00000000  00000000 0000000d  ................\\nruntime: g 0 gp=0x170ac5c1f0e0: unknown pc 0x7f7ef9baa94c\\nstack: frame={sp:0x7f7eabffe790, fp:0x0} stack=[0x7f7eab7ff1e0,0x7f7eabffede0)\\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\\n00007f7eabffe690: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6a0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6b0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6c0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6d0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6e0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6f0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe700: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe710: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe720: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe730: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe740: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe750: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe760: 00000000 00000000  00007f7e f9bb7fa9  ............~...\\n00007f7eabffe770: 00000000 00000000  00000000 00000011  ................\\n00007f7eabffe780: 00000000 00000130  00007f7e f9baa93e  0.......\u003e...~...\\n00007f7eabffe790: 00000000 00000120  00000000 00000000   ...............\\n                  ^ SP\\n00007f7eabffe7a0: 00000000 00000013  00000000 00000000  ................\\n00007f7eabffe7b0: 00000000 00000000  00007f7e f9c2dcff  ............~...\\n00007f7eabffe7c0: 00007f7e 7bfff910  00007f7e f9c2dcb1  ...{~.......~...\\n00007f7eabffe7d0: 00007f7e 7bfff640  00007f7e abffeaa0  @..{~.......~...\\n00007f7eabffe7e0: 00007f7e abffe90e  00007f7e abffe90f  ....~.......~...\\n00007f7eabffe7f0: 00007f7e 7bfff640  00007f7e f9ba87f5  @..{~.......~...\\n00007f7eabffe800: 00007f7e a4000020  00000000 00000120   ...~... .......\\n00007f7eabffe810: 00000000 003d0f00  176a34c4 bac87500  ..=......u...4j.\\n00007f7eabffe820: 00007f7e abfff640  00000000 00000006  @...~...........\\n00007f7eabffe830: 00007f7e a4000f30  00007f7e abffeb60  0...~...`...~...\\n00007f7eabffe840: 0000170a c5c1f0e0  00007f7e f9b5d646  ........F...~...\\n00007f7eabffe850: 00007f7e f9d1ae90  00007f7e f9b477f3  ....~....w..~...\\n00007f7eabffe860: 00000000 00000020  176a34c4 bac87500   ........u...4j.\\n00007f7eabffe870: 00007f7e 7b7ff000  00007f7e f9c1cac4  ...{~.......~...\\n00007f7eabffe880: 00000000 00000000  00000000 0000000d  ................\\n\\ngoroutine 1 gp=0x170ac5c1e1e0 m=nil [sync.WaitGroup.Wait]:\\nruntime.gopark(0x170ac5cb27b8?, 0x2022a4d?, 0x70?, 0x8e?, 0x170ac5cb27e8?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cb27a0 sp=0x170ac5cb2780 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x170ac65f8210, 0x0, 0x1, 0x0, 0x19)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x170ac5cb2808 sp=0x170ac5cb27a0 pc=0x1f162b2\\nsync.runtime_SemacquireWaitGroup(0x170ac6709950?, 0xe0?)\\n\\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x170ac5cb2840 sp=0x170ac5cb2808 pc=0x1f3a66e\\nsync.(*WaitGroup).Wait(0x170ac65f8208)\\n\\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x170ac5cb2868 sp=0x170ac5cb2840 pc=0x1f4f445\\ngolang.org/x/sync/errgroup.(*Group).Wait(0x170ac65f8200)\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x170ac5cb2888 sp=0x170ac5cb2868 pc=0x362507e\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0}, {0x170ac626a1e0, 0x6, 0x6})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x170ac5cb2948 sp=0x170ac5cb2888 pc=0x3927e05\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).gatherInfraLogs(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:595 +0xadd fp=0x170ac5cb2d30 sp=0x170ac5cb2948 pc=0x392799d\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:333 +0x533 fp=0x170ac5cb3908 sp=0x170ac5cb2d30 pc=0x3921bf3\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedInfraQueryOptions).RunInfra(0x170ac5c2c108, {0x59f0b08, 0x170ac64108d0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:83 +0x71f fp=0x170ac5cb3b30 sp=0x170ac5cb3908 pc=0x393887f\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawInfraQueryOptions).Run(0x0?, {0x59f0b08, 0x170ac64108d0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_options.go:109 +0x54 fp=0x170ac5cb3b58 sp=0x170ac5cb3b30 pc=0x3939134\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryInfraCommand.func1(0x170ac66f5508?, {0x170ac5d40960?, 0x4?, 0x553b0e2?})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:45 +0x28 fp=0x170ac5cb3b80 sp=0x170ac5cb3b58 pc=0x3938128\\ngithub.com/spf13/cobra.(*Command).execute(0x170ac66f5508, {0x170ac5e36100, 0xa, 0xa})\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x170ac5cb3d28 sp=0x170ac5cb3b80 pc=0x212e614\\ngithub.com/spf13/cobra.(*Command).ExecuteC(0x170ac6228608)\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x170ac5cb3e18 sp=0x170ac5cb3d28 pc=0x212efa5\\ngithub.com/spf13/cobra.(*Command).Execute(...)\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071\\ngithub.com/spf13/cobra.(*Command).ExecuteContext(...)\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064\\nmain.main()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x170ac5cb3f48 sp=0x170ac5cb3e18 pc=0x47ddd6a\\nruntime.main()\\n\\t/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x170ac5cb3fe0 sp=0x170ac5cb3f48 pc=0x1f01975\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cb3fe8 sp=0x170ac5cb3fe0 pc=0x1f40801\\n\\ngoroutine 2 gp=0x170ac5c1ed20 m=nil [force gc (idle)]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdefa8 sp=0x170ac5cdef88 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.forcegchelper()\\n\\t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x170ac5cdefe0 sp=0x170ac5cdefa8 pc=0x1f01c93\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdefe8 sp=0x170ac5cdefe0 pc=0x1f40801\\ncreated by runtime.init.7 in goroutine 1\\n\\t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\\n\\ngoroutine 18 gp=0x170ac5d843c0 m=nil [GC sweep wait]:\\nruntime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cda788 sp=0x170ac5cda768 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.bgsweep(0x170ac5d92000)\\n\\t/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x170ac5cda7c8 sp=0x170ac5cda788 pc=0x1ee9f11\\nruntime.gcenable.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x170ac5cda7e0 sp=0x170ac5cda7c8 pc=0x1edadd7\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cda7e8 sp=0x170ac5cda7e0 pc=0x1f40801\\ncreated by runtime.gcenable in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\\n\\ngoroutine 19 gp=0x170ac5d845a0 m=nil [GC scavenge wait]:\\nruntime.gopark(0x10000?, 0x5981cd8?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdaf78 sp=0x170ac5cdaf58 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.(*scavengerState).park(0x82979c0)\\n\\t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x170ac5cdafa8 sp=0x170ac5cdaf78 pc=0x1ee77e9\\nruntime.bgscavenge(0x170ac5d92000)\\n\\t/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x170ac5cdafc8 sp=0x170ac5cdafa8 pc=0x1ee7d99\\nruntime.gcenable.gowrap2()\\n\\t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x170ac5cdafe0 sp=0x170ac5cdafc8 pc=0x1edad97\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdafe8 sp=0x170ac5cdafe0 pc=0x1f40801\\ncreated by runtime.gcenable in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\\n\\ngoroutine 34 gp=0x170ac5e00000 m=nil [GOMAXPROCS updater (idle)]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e06788 sp=0x170ac5e06768 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.updateMaxProcsGoroutine()\\n\\t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x170ac5e067e0 sp=0x170ac5e06788 pc=0x1f10267\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e067e8 sp=0x170ac5e067e0 pc=0x1f40801\\ncreated by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\\n\\t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\\n\\ngoroutine 35 gp=0x170ac5e005a0 m=nil [finalizer wait]:\\nruntime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x170ac5cde601?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cde620 sp=0x170ac5cde600 pc=0x1f38d2e\\nruntime.runFinalizers()\\n\\t/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x170ac5cde7e0 sp=0x170ac5cde620 pc=0x1ed9cc7\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cde7e8 sp=0x170ac5cde7e0 pc=0x1f40801\\ncreated by runtime.createfing in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d\\n\\ngoroutine 36 gp=0x170ac5ff81e0 m=nil [cleanup wait]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e06f68 sp=0x170ac5e06f48 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.(*cleanupQueue).dequeue(0x82980e0)\\n\\t/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x170ac5e06fa0 sp=0x170ac5e06f68 pc=0x1ed6074\\nruntime.runCleanups()\\n\\t/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x170ac5e06fe0 sp=0x170ac5e06fa0 pc=0x1ed66e5\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e06fe8 sp=0x170ac5e06fe0 pc=0x1f40801\\ncreated by runtime.(*cleanupQueue).createGs in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5\\n\\ngoroutine 37 gp=0x170ac611bc20 m=nil [GC worker (idle)]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e07740 sp=0x170ac5e07720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5e077c8 sp=0x170ac5e07740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5e077e0 sp=0x170ac5e077c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e077e8 sp=0x170ac5e077e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 3 gp=0x170ac5c1f680 m=nil [runnable]:\\nruntime.gopark(0x125c26559f5b?, 0x3?, 0x94?, 0x85?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdf740 sp=0x170ac5cdf720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdf7c8 sp=0x170ac5cdf740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdf7e0 sp=0x170ac5cdf7c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdf7e8 sp=0x170ac5cdf7e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 20 gp=0x170ac5d84780 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125c265c3db1?, 0x3?, 0x92?, 0xcd?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdb740 sp=0x170ac5cdb720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdb7c8 sp=0x170ac5cdb740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdb7e0 sp=0x170ac5cdb7c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdb7e8 sp=0x170ac5cdb7e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 38 gp=0x170ac6134000 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125c26588b9e?, 0x1?, 0xf7?, 0xb4?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e07f40 sp=0x170ac5e07f20 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5e07fc8 sp=0x170ac5e07f40 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5e07fe0 sp=0x170ac5e07fc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e07fe8 sp=0x170ac5e07fe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 4 gp=0x170ac5c1f860 m=10 mp=0x170ac6280008 [GC mark termination]:\\nruntime.systemstack_switch()\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x170ac5cdfea0 sp=0x170ac5cdfe90 pc=0x1f3ebc8\\nruntime.forEachP(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:2112\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x170ac5cdff40 sp=0x170ac5cdfea0 pc=0x1edbc34\\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x170ac5cdffc8 sp=0x170ac5cdff40 pc=0x1edda88\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdffe0 sp=0x170ac5cdffc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdffe8 sp=0x170ac5cdffe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 21 gp=0x170ac5d84960 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125c265818cf?, 0x3?, 0xa5?, 0x9c?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdbf40 sp=0x170ac5cdbf20 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdbfc8 sp=0x170ac5cdbf40 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdbfe0 sp=0x170ac5cdbfc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdbfe8 sp=0x170ac5cdbfe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 22 gp=0x170ac5d84b40 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125c265846e7?, 0x3?, 0x77?, 0xb6?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdc740 sp=0x170ac5cdc720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdc7c8 sp=0x170ac5cdc740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdc7e0 sp=0x170ac5cdc7c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdc7e8 sp=0x170ac5cdc7e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 23 gp=0x170ac5d84d20 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125c26581951?, 0x3?, 0x91?, 0x58?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdcf40 sp=0x170ac5cdcf20 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdcfc8 sp=0x170ac5cdcf40 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdcfe0 sp=0x170ac5cdcfc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdcfe8 sp=0x170ac5cdcfe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 39 gp=0x170ac648a3c0 m=nil [select, locked to thread]:\\nruntime.gopark(0x170ac5e097a8?, 0x2?, 0x1?, 0xa3?, 0x170ac5e09794?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e09628 sp=0x170ac5e09608 pc=0x1f38d2e\\nruntime.selectgo(0x170ac5e097a8, 0x170ac5e09790, 0x0?, 0x0, 0x0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5e09758 sp=0x170ac5e09628 pc=0x1f152a5\\nruntime.ensureSigM.func1()\\n\\t/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x170ac5e097e0 sp=0x170ac5e09758 pc=0x1f328e8\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e097e8 sp=0x170ac5e097e0 pc=0x1f40801\\ncreated by runtime.ensureSigM in goroutine 1\\n\\t/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5\\n\\ngoroutine 24 gp=0x170ac62f4960 m=8 mp=0x170ac6180008 [syscall]:\\nruntime.notetsleepg(0x82bdea0, 0xffffffffffffffff)\\n\\t/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x170ac5e037a0 sp=0x170ac5e03778 pc=0x1ece509\\nos/signal.signal_recv()\\n\\t/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x170ac5e037c0 sp=0x170ac5e037a0 pc=0x1f3b298\\nos/signal.loop()\\n\\t/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x170ac5e037e0 sp=0x170ac5e037c0 pc=0x2030553\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e037e8 sp=0x170ac5e037e0 pc=0x1f40801\\ncreated by os/signal.Notify.func1.1 in goroutine 1\\n\\t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\\n\\ngoroutine 50 gp=0x170ac6135a40 m=nil [select]:\\nruntime.gopark(0x170ac66e3798?, 0x2?, 0x80?, 0x6f?, 0x170ac66e378c?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac66e3628 sp=0x170ac66e3608 pc=0x1f38d2e\\nruntime.selectgo(0x170ac66e3798, 0x170ac66e3788, 0x0?, 0x0, 0x0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac66e3758 sp=0x170ac66e3628 pc=0x1f152a5\\nos/signal.NotifyContext.func1()\\n\\t/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x170ac66e37e0 sp=0x170ac66e3758 pc=0x202ff9c\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac66e37e8 sp=0x170ac66e37e0 pc=0x1f40801\\ncreated by os/signal.NotifyContext in goroutine 1\\n\\t/usr/lib/golang/src/os/signal/signal.go:291 +0x158\\n\\ngoroutine 102 gp=0x170ac61843c0 m=nil [select]:\\nruntime.gopark(0x170ac5cefc70?, 0x2?, 0xc8?, 0xf9?, 0x170ac5cefb6c?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cef9f0 sp=0x170ac5cef9d0 pc=0x1f38d2e\\nruntime.selectgo(0x170ac5cefc70, 0x170ac5cefb68, 0x554857c?, 0x0, 0x226?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5cefb20 sp=0x170ac5cef9f0 pc=0x1f152a5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x170ac5c49130}, 0x170ac65fa070, 0x170ac6410cf0)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:283 +0x21b fp=0x170ac5ceff48 sp=0x170ac5cefb20 pc=0x39204fb\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x170ac5ceff78 sp=0x170ac5ceff48 pc=0x392800c\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5ceffe0 sp=0x170ac5ceff78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5ceffe8 sp=0x170ac5ceffe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 104 gp=0x170ac6184d20 m=nil [select]:\\nruntime.gopark(0x170ac6992ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69929d8?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6992848 sp=0x170ac6992828 pc=0x1f38d2e\\nruntime.selectgo(0x170ac6992ac8, 0x170ac69929d0, 0x55562e1?, 0x0, 0x1f6d6cf?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6992978 sp=0x170ac6992848 pc=0x1f152a5\\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac619fb80, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac6992b40 sp=0x170ac6992978 pc=0x22d92e5\\nnet/http.(*http2ClientConn).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac619fb80, {0x4c?, 0xfa?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac6992c80 sp=0x170ac6992b40 pc=0x22d50ad\\nnet/http.(*http2Transport).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac619fb80?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac6992cc0 sp=0x170ac6992c80 pc=0x22e3a9c\\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac619fa40)\\n\\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac6992ea0 sp=0x170ac6992cc0 pc=0x231037f\\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\\n\\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac6992ec0 sp=0x170ac6992ea0 pc=0x2321178\\nnet/http.send(0x170ac619fa40, {0x59a1ce0, 0x8175400}, {0x170ac6993138?, 0x1f3bac6?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69930b0 sp=0x170ac6992ec0 pc=0x22ad9ab\\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac619fa40, {0x170ac69931e8?, 0x1fdd787?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac6993148 sp=0x170ac69930b0 pc=0x22ad1d8\\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac619fa40)\\n\\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac6993338 sp=0x170ac6993148 pc=0x22af817\\nnet/http.(*Client).Do(...)\\n\\t/usr/lib/golang/src/net/http/client.go:592\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5c49450}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e34a0}, 0x170ac65f7ce0, {0x170ac65d9560, 0x112})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac6993630 sp=0x170ac6993338 pc=0x38de6bd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5c49450}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x170ac65f7c80, 0x0, {0x0, ...}, ...})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac6993888 sp=0x170ac6993630 pc=0x38ddecd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5c49450?}, 0xa8?, {0x170ac5db1a50?, 0x170ac6764a48?}, 0x3?, 0x2022a4d?)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac6993960 sp=0x170ac6993888 pc=0x38dd40b\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194700?}, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x170ac6764a48, 0x3, 0x4})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69939e0 sp=0x170ac6993960 pc=0x38e51e2\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194700}, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x0?, 0x2?, 0x1?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac6993a78 sp=0x170ac69939e0 pc=0x38e4eaf\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020500}, 0x170ac65fa000)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac6993d60 sp=0x170ac6993a78 pc=0x391b5c5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac5fe1880?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020500?}, 0x0?)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac6993e88 sp=0x170ac6993d60 pc=0x3929c16\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac6993f78 sp=0x170ac6993e88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac6993fe0 sp=0x170ac6993f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6993fe8 sp=0x170ac6993fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 101 gp=0x170ac61d4000 m=nil [sync.WaitGroup.Wait]:\\nruntime.gopark(0x170ac5de7e00?, 0x1ed0765?, 0xb0?, 0xfa?, 0xe?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de7da0 sp=0x170ac5de7d80 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x170ac65f8290, 0x0, 0x1, 0x0, 0x19)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x170ac5de7e08 sp=0x170ac5de7da0 pc=0x1f162b2\\nsync.runtime_SemacquireWaitGroup(0x170ac6709a70?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x170ac5de7e40 sp=0x170ac5de7e08 pc=0x1f3a66e\\nsync.(*WaitGroup).Wait(0x170ac65f8288)\\n\\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x170ac5de7e68 sp=0x170ac5de7e40 pc=0x1f4f445\\ngolang.org/x/sync/errgroup.(*Group).Wait(0x170ac65f8280)\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x170ac5de7e88 sp=0x170ac5de7e68 pc=0x362507e\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x170ac5fe1140, {0x59f0b40, 0x170ac5c49130}, {0x170ac626a1e0, 0x6, 0x170ac5cac1c0?}, 0x170ac65fa000)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x170ac5de7f00 sp=0x170ac5de7e88 pc=0x392976a\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x170ac5de7f78 sp=0x170ac5de7f00 pc=0x392808e\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5de7fe0 sp=0x170ac5de7f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de7fe8 sp=0x170ac5de7fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 67 gp=0x170ac62f5a40 m=nil [IO wait]:\\nruntime.gopark(0x170ac6497780?, 0x8?, 0x40?, 0x16?, 0xb?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de47b8 sp=0x170ac5de4798 pc=0x1f38d2e\\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x170ac5de47f0 sp=0x170ac5de47b8 pc=0x1ef9e57\\ninternal/poll.runtime_pollWait(0x7f7eb1670000, 0x72)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x170ac5de4810 sp=0x170ac5de47f0 pc=0x1f375c5\\ninternal/poll.(*pollDesc).wait(0x170ac620b480?, 0x170ac65a2000?, 0x0)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x170ac5de4838 sp=0x170ac5de4810 pc=0x1fbd167\\ninternal/poll.(*pollDesc).waitRead(...)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\\ninternal/poll.(*FD).Read(0x170ac620b480, {0x170ac65a2000, 0x3500, 0x3500})\\n\\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x170ac5de48d0 sp=0x170ac5de4838 pc=0x1fbe4ce\\nnet.(*netFD).Read(0x170ac620b480, {0x170ac65a2000?, 0x170ac5de4901?, 0x22a2e75?})\\n\\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x170ac5de4918 sp=0x170ac5de48d0 pc=0x20798c5\\nnet.(*conn).Read(0x170ac62823d8, {0x170ac65a2000?, 0x7f7eb047fc00?, 0x7f7ef9af2e00?})\\n\\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x170ac5de4960 sp=0x170ac5de4918 pc=0x208b7a5\\ncrypto/tls.(*atLeastReader).Read(0x170ac6708f30, {0x170ac65a2000?, 0x170ac5de4a48?, 0x1efcc28?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x170ac5de49a8 sp=0x170ac5de4960 pc=0x2222ddb\\nbytes.(*Buffer).ReadFrom(0x170ac605fb28, {0x59a3680, 0x170ac6708f30})\\n\\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x170ac5de4a00 sp=0x170ac5de49a8 pc=0x1fe9258\\ncrypto/tls.(*Conn).readFromUntil(0x170ac605f888, {0x59a32c0, 0x170ac62823d8}, 0x222764c?)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x170ac5de4a38 sp=0x170ac5de4a00 pc=0x2222fbe\\ncrypto/tls.(*Conn).readRecordOrCCS(0x170ac605f888, 0x0)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x170ac5de4c18 sp=0x170ac5de4a38 pc=0x222009b\\ncrypto/tls.(*Conn).readRecord(...)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:588\\ncrypto/tls.(*Conn).Read(0x170ac605f888, {0x170ac6029000, 0x1000, 0x22c34e0?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x170ac5de4c80 sp=0x170ac5de4c18 pc=0x22267c5\\nbufio.(*Reader).Read(0x170ac647aa20, {0x170ac6153624, 0x9, 0x22e100e?})\\n\\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0x170ac5de4cb8 sp=0x170ac5de4c80 pc=0x20a71f7\\nio.ReadAtLeast({0x59a1260, 0x170ac647aa20}, {0x170ac6153624, 0x9, 0x9}, 0x9)\\n\\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0x170ac5de4d00 sp=0x170ac5de4cb8 pc=0x1fb17ee\\nio.ReadFull(...)\\n\\t/usr/lib/golang/src/io/io.go:354\\nnet/http.http2readFrameHeader({0x170ac6153624, 0x9, 0x170a00000005?}, {0x59a1260?, 0x170ac647aa20?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:1805 +0x65 fp=0x170ac5de4d50 sp=0x170ac5de4d00 pc=0x22b9e25\\nnet/http.(*http2Framer).ReadFrameHeader(0x170ac61535e0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:2071 +0x6b fp=0x170ac5de4dd8 sp=0x170ac5de4d50 pc=0x22ba9eb\\nnet/http.(*http2Framer).ReadFrame(0x170ac61535e0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:2130 +0x18 fp=0x170ac5de4e18 sp=0x170ac5de4dd8 pc=0x22bb1b8\\nnet/http.(*http2clientConnReadLoop).run(0x170ac5de4fa8)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9550 +0xca fp=0x170ac5de4f60 sp=0x170ac5de4e18 pc=0x22de14a\\nnet/http.(*http2ClientConn).readLoop(0x170ac61d9880)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x170ac5de4fc8 sp=0x170ac5de4f60 pc=0x22dd532\\nnet/http.(*http2Transport).newClientConn.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x170ac5de4fe0 sp=0x170ac5de4fc8 pc=0x22d7057\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de4fe8 sp=0x170ac5de4fe0 pc=0x1f40801\\ncreated by net/http.(*http2Transport).newClientConn in goroutine 66\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5\\n\\ngoroutine 73 gp=0x170ac648ad20 m=nil [IO wait]:\\nruntime.gopark(0x170ac6497e40?, 0x1edb176?, 0x80?, 0xed?, 0xb?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de6808 sp=0x170ac5de67e8 pc=0x1f38d2e\\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x170ac5de6840 sp=0x170ac5de6808 pc=0x1ef9e57\\ninternal/poll.runtime_pollWait(0x7f7eb166fe00, 0x72)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x170ac5de6860 sp=0x170ac5de6840 pc=0x1f375c5\\ninternal/poll.(*pollDesc).wait(0x170ac60c4e80?, 0x170ac5ecd300?, 0x0)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x170ac5de6888 sp=0x170ac5de6860 pc=0x1fbd167\\ninternal/poll.(*pollDesc).waitRead(...)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\\ninternal/poll.(*FD).Read(0x170ac60c4e80, {0x170ac5ecd300, 0x1300, 0x1300})\\n\\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x170ac5de6920 sp=0x170ac5de6888 pc=0x1fbe4ce\\nnet.(*netFD).Read(0x170ac60c4e80, {0x170ac5ecd300?, 0x170ac5ecd30a?, 0x5?})\\n\\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x170ac5de6968 sp=0x170ac5de6920 pc=0x20798c5\\nnet.(*conn).Read(0x170ac5d82278, {0x170ac5ecd300?, 0x7f7eb047fc00?, 0x7f7ef9af2e00?})\\n\\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x170ac5de69b0 sp=0x170ac5de6968 pc=0x208b7a5\\ncrypto/tls.(*atLeastReader).Read(0x170ac67088e8, {0x170ac5ecd300?, 0x170ac5de6a98?, 0x1efcc28?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x170ac5de69f8 sp=0x170ac5de69b0 pc=0x2222ddb\\nbytes.(*Buffer).ReadFrom(0x170ac5dd02a8, {0x59a3680, 0x170ac67088e8})\\n\\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x170ac5de6a50 sp=0x170ac5de69f8 pc=0x1fe9258\\ncrypto/tls.(*Conn).readFromUntil(0x170ac5dd0008, {0x59a32c0, 0x170ac5d82278}, 0x222764c?)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x170ac5de6a88 sp=0x170ac5de6a50 pc=0x2222fbe\\ncrypto/tls.(*Conn).readRecordOrCCS(0x170ac5dd0008, 0x0)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x170ac5de6c68 sp=0x170ac5de6a88 pc=0x222009b\\ncrypto/tls.(*Conn).readRecord(...)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:588\\ncrypto/tls.(*Conn).Read(0x170ac5dd0008, {0x170ac6564000, 0x1000, 0x4e55540?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x170ac5de6cd0 sp=0x170ac5de6c68 pc=0x22267c5\\nnet/http.(*persistConn).Read(0x170ac5f73180, {0x170ac6564000?, 0x59a0ea0?, 0x81260e0?})\\n\\t/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x170ac5de6d30 sp=0x170ac5de6cd0 pc=0x2319d07\\nbufio.(*Reader).fill(0x170ac65537a0)\\n\\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x170ac5de6d68 sp=0x170ac5de6d30 pc=0x20a6c63\\nbufio.(*Reader).Peek(0x170ac65537a0, 0x1)\\n\\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x170ac5de6d80 sp=0x170ac5de6d68 pc=0x20a6df3\\nnet/http.(*persistConn).readLoop(0x170ac5f73180)\\n\\t/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x170ac5de6fc8 sp=0x170ac5de6d80 pc=0x231a812\\nnet/http.(*Transport).dialConn.gowrap2()\\n\\t/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x170ac5de6fe0 sp=0x170ac5de6fc8 pc=0x2319197\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de6fe8 sp=0x170ac5de6fe0 pc=0x1f40801\\ncreated by net/http.(*Transport).dialConn in goroutine 82\\n\\t/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac\\n\\ngoroutine 103 gp=0x170ac648af00 m=nil [select]:\\nruntime.gopark(0x170ac5de0d00?, 0x2?, 0xa0?, 0x6a?, 0x170ac5de0c74?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de0ac8 sp=0x170ac5de0aa8 pc=0x1f38d2e\\nruntime.selectgo(0x170ac5de0d00, 0x170ac5de0c70, 0x553bfdc?, 0x0, 0x5a48c80?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5de0bf8 sp=0x170ac5de0ac8 pc=0x1f152a5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x0?, {0x59f0b40, 0x170ac5c49130}, 0x170ac65fa000, 0x170ac65fa070)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x170ac5de0f10 sp=0x170ac5de0bf8 pc=0x392836e\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x170ac5de0f78 sp=0x170ac5de0f10 pc=0x3927f3d\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5de0fe0 sp=0x170ac5de0f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de0fe8 sp=0x170ac5de0fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 74 gp=0x170ac648b4a0 m=nil [select]:\\nruntime.gopark(0x170ac5df1f48?, 0x2?, 0x0?, 0xb6?, 0x170ac5df1ef4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6760d88 sp=0x170ac6760d68 pc=0x1f38d2e\\nruntime.selectgo(0x170ac6760f48, 0x170ac5df1ef0, 0x170ac68d6980?, 0x0, 0x170ac655e8d0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6760eb8 sp=0x170ac6760d88 pc=0x1f152a5\\nnet/http.(*persistConn).writeLoop(0x170ac5f73180)\\n\\t/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x170ac6760fc8 sp=0x170ac6760eb8 pc=0x231c426\\nnet/http.(*Transport).dialConn.gowrap3()\\n\\t/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x170ac6760fe0 sp=0x170ac6760fc8 pc=0x2319157\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6760fe8 sp=0x170ac6760fe0 pc=0x1f40801\\ncreated by net/http.(*Transport).dialConn in goroutine 82\\n\\t/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe\\n\\ngoroutine 105 gp=0x170ac61d43c0 m=nil [select]:\\nruntime.gopark(0x170ac699cac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac699c9d8?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac699c848 sp=0x170ac699c828 pc=0x1f38d2e\\nruntime.selectgo(0x170ac699cac8, 0x170ac699c9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac699c978 sp=0x170ac699c848 pc=0x1f152a5\\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e000, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac699cb40 sp=0x170ac699c978 pc=0x22d92e5\\nnet/http.(*http2ClientConn).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e000, {0x4c?, 0xfa?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac699cc80 sp=0x170ac699cb40 pc=0x22d50ad\\nnet/http.(*http2Transport).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e000?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac699ccc0 sp=0x170ac699cc80 pc=0x22e3a9c\\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac619fcc0)\\n\\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac699cea0 sp=0x170ac699ccc0 pc=0x231037f\\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\\n\\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac699cec0 sp=0x170ac699cea0 pc=0x2321178\\nnet/http.send(0x170ac619fcc0, {0x59a1ce0, 0x8175400}, {0x170ac699d138?, 0x1f3bac6?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac699d0b0 sp=0x170ac699cec0 pc=0x22ad9ab\\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac619fcc0, {0x170ac699d1e8?, 0x1fdd787?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac699d148 sp=0x170ac699d0b0 pc=0x22ad1d8\\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac619fcc0)\\n\\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac699d338 sp=0x170ac699d148 pc=0x22af817\\nnet/http.(*Client).Do(...)\\n\\t/usr/lib/golang/src/net/http/client.go:592\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e000}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e36b0}, 0x170ac65f7e00, {0x170ac65d9680, 0x118})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac699d630 sp=0x170ac699d338 pc=0x38de6bd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e000}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x170ac65f7da0, 0x0, {0x0, ...}, ...})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac699d888 sp=0x170ac699d630 pc=0x38ddecd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e000?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de1a48?}, 0x3?, 0x2022a4d?)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac699d960 sp=0x170ac699d888 pc=0x38dd40b\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194850?}, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x170ac5de1a48, 0x3, 0x4})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac699d9e0 sp=0x170ac699d960 pc=0x38e51e2\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194850}, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x0?, 0x2?, 0x0?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac699da78 sp=0x170ac699d9e0 pc=0x38e4eaf\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac60207c0}, 0x170ac65fa000)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac699dd60 sp=0x170ac699da78 pc=0x391b5c5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac60207c0?}, 0x0?)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac699de88 sp=0x170ac699dd60 pc=0x3929c16\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac699df78 sp=0x170ac699de88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac699dfe0 sp=0x170ac699df78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac699dfe8 sp=0x170ac699dfe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 107 gp=0x170ac61d45a0 m=nil [select]:\\nruntime.gopark(0x170ac69b2ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69b29d8?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69b2848 sp=0x170ac69b2828 pc=0x1f38d2e\\nruntime.selectgo(0x170ac69b2ac8, 0x170ac69b29d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69b2978 sp=0x170ac69b2848 pc=0x1f152a5\\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e500, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69b2b40 sp=0x170ac69b2978 pc=0x22d92e5\\nnet/http.(*http2ClientConn).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e500, {0x4c?, 0xfa?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69b2c80 sp=0x170ac69b2b40 pc=0x22d50ad\\nnet/http.(*http2Transport).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e500?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69b2cc0 sp=0x170ac69b2c80 pc=0x22e3a9c\\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac699e3c0)\\n\\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69b2ea0 sp=0x170ac69b2cc0 pc=0x231037f\\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\\n\\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69b2ec0 sp=0x170ac69b2ea0 pc=0x2321178\\nnet/http.send(0x170ac699e3c0, {0x59a1ce0, 0x8175400}, {0x170ac69b3138?, 0x1f3bac6?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69b30b0 sp=0x170ac69b2ec0 pc=0x22ad9ab\\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac699e3c0, {0x4d3aa8?, 0xcd6610?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69b3148 sp=0x170ac69b30b0 pc=0x22ad1d8\\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac699e3c0)\\n\\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69b3338 sp=0x170ac69b3148 pc=0x22af817\\nnet/http.(*Client).Do(...)\\n\\t/usr/lib/golang/src/net/http/client.go:592\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e1e0}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3a60}, 0x170ac69a0120, {0x170ac65d98c0, 0x111})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69b3630 sp=0x170ac69b3338 pc=0x38de6bd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e1e0}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x170ac69a00c0, 0x0, {0x0, ...}, ...})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69b3888 sp=0x170ac69b3630 pc=0x38ddecd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e1e0?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de2a48?}, 0x3?, 0x2022a4d?)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69b3960 sp=0x170ac69b3888 pc=0x38dd40b\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194bd0?}, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x170ac5de2a48, 0x3, 0x4})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69b39e0 sp=0x170ac69b3960 pc=0x38e51e2\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194bd0}, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x0?, 0x2?, 0x0?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69b3a78 sp=0x170ac69b39e0 pc=0x38e4eaf\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020ec0}, 0x170ac65fa000)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69b3d60 sp=0x170ac69b3a78 pc=0x391b5c5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020ec0?}, 0x0?)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69b3e88 sp=0x170ac69b3d60 pc=0x3929c16\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69b3f78 sp=0x170ac69b3e88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69b3fe0 sp=0x170ac69b3f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69b3fe8 sp=0x170ac69b3fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 106 gp=0x170ac648b680 m=nil [select]:\\nruntime.gopark(0x170ac69aaac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69aa9d8?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69aa848 sp=0x170ac69aa828 pc=0x1f38d2e\\nruntime.selectgo(0x170ac69aaac8, 0x170ac69aa9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69aa978 sp=0x170ac69aa848 pc=0x1f152a5\\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e280, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69aab40 sp=0x170ac69aa978 pc=0x22d92e5\\nnet/http.(*http2ClientConn).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e280, {0x4c?, 0xfa?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69aac80 sp=0x170ac69aab40 pc=0x22d50ad\\nnet/http.(*http2Transport).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e280?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69aacc0 sp=0x170ac69aac80 pc=0x22e3a9c\\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac699e140)\\n\\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69aaea0 sp=0x170ac69aacc0 pc=0x231037f\\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\\n\\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69aaec0 sp=0x170ac69aaea0 pc=0x2321178\\nnet/http.send(0x170ac699e140, {0x59a1ce0, 0x8175400}, {0x170ac69ab138?, 0x1f3bac6?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69ab0b0 sp=0x170ac69aaec0 pc=0x22ad9ab\\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac699e140, {0x170ac69ab1e8?, 0x1fdd787?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69ab148 sp=0x170ac69ab0b0 pc=0x22ad1d8\\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac699e140)\\n\\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69ab338 sp=0x170ac69ab148 pc=0x22af817\\nnet/http.(*Client).Do(...)\\n\\t/usr/lib/golang/src/net/http/client.go:592\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e0f0}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e38c0}, 0x170ac69a0030, {0x170ac65d97a0, 0x10e})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69ab630 sp=0x170ac69ab338 pc=0x38de6bd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e0f0}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x170ac65f7ec0, 0x0, {0x0, ...}, ...})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69ab888 sp=0x170ac69ab630 pc=0x38ddecd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e0f0?}, 0xa8?, {0x170ac5db1a50?, 0x170ac6766a48?}, 0x3?, 0x2022a4d?)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69ab960 sp=0x170ac69ab888 pc=0x38dd40b\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194a80?}, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x170ac6766a48, 0x3, 0x4})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69ab9e0 sp=0x170ac69ab960 pc=0x38e51e2\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194a80}, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x0?, 0x2?, 0x0?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69aba78 sp=0x170ac69ab9e0 pc=0x38e4eaf\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020cc0}, 0x170ac65fa000)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69abd60 sp=0x170ac69aba78 pc=0x391b5c5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac6766ea0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020cc0?}, 0x170ac5f6f3b0?)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69abe88 sp=0x170ac69abd60 pc=0x3929c16\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69abf78 sp=0x170ac69abe88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69abfe0 sp=0x170ac69abf78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69abfe8 sp=0x170ac69abfe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 108 gp=0x170ac61d4780 m=nil [select]:\\nruntime.gopark(0x170ac69c0ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69c09d8?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69c0848 sp=0x170ac69c0828 pc=0x1f38d2e\\nruntime.selectgo(0x170ac69c0ac8, 0x170ac69c09d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69c0978 sp=0x170ac69c0848 pc=0x1f152a5\\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e780, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69c0b40 sp=0x170ac69c0978 pc=0x22d92e5\\nnet/http.(*http2ClientConn).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e780, {0x4c?, 0xfa?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69c0c80 sp=0x170ac69c0b40 pc=0x22d50ad\\nnet/http.(*http2Transport).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e780?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69c0cc0 sp=0x170ac69c0c80 pc=0x22e3a9c\\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac699e640)\\n\\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69c0ea0 sp=0x170ac69c0cc0 pc=0x231037f\\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\\n\\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69c0ec0 sp=0x170ac69c0ea0 pc=0x2321178\\nnet/http.send(0x170ac699e640, {0x59a1ce0, 0x8175400}, {0x170ac69c1138?, 0x1f3bac6?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69c10b0 sp=0x170ac69c0ec0 pc=0x22ad9ab\\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac699e640, {0x82bed80?, 0x6401fdd700?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69c1148 sp=0x170ac69c10b0 pc=0x22ad1d8\\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac699e640)\\n\\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69c1338 sp=0x170ac69c1148 pc=0x22af817\\nnet/http.(*Client).Do(...)\\n\\t/usr/lib/golang/src/net/http/client.go:592\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e280}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3bf0}, 0x170ac69a0210, {0x170ac65d99e0, 0x110})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69c1630 sp=0x170ac69c1338 pc=0x38de6bd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e280}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x170ac69a01b0, 0x0, {0x0, ...}, ...})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69c1888 sp=0x170ac69c1630 pc=0x38ddecd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e280?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de2a48?}, 0x3?, 0x2022a4d?)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69c1960 sp=0x170ac69c1888 pc=0x38dd40b\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194d90?}, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x170ac5de2a48, 0x3, 0x4})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69c19e0 sp=0x170ac69c1960 pc=0x38e51e2\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194d90}, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x0?, 0x2?, 0x0?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69c1a78 sp=0x170ac69c19e0 pc=0x38e4eaf\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6021d80}, 0x170ac65fa000)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69c1d60 sp=0x170ac69c1a78 pc=0x391b5c5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac66e1d6a?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6021d80?}, 0x0?)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69c1e88 sp=0x170ac69c1d60 pc=0x3929c16\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69c1f78 sp=0x170ac69c1e88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69c1fe0 sp=0x170ac69c1f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69c1fe8 sp=0x170ac69c1fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 109 gp=0x170ac61d4960 m=nil [select]:\\nruntime.gopark(0x170ac69caac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69ca9d8?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69ca848 sp=0x170ac69ca828 pc=0x1f38d2e\\nruntime.selectgo(0x170ac69caac8, 0x170ac69ca9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69ca978 sp=0x170ac69ca848 pc=0x1f152a5\\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699ea00, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69cab40 sp=0x170ac69ca978 pc=0x22d92e5\\nnet/http.(*http2ClientConn).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699ea00, {0x4c?, 0xfa?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69cac80 sp=0x170ac69cab40 pc=0x22d50ad\\nnet/http.(*http2Transport).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699ea00?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69cacc0 sp=0x170ac69cac80 pc=0x22e3a9c\\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac699e8c0)\\n\\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69caea0 sp=0x170ac69cacc0 pc=0x231037f\\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\\n\\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69caec0 sp=0x170ac69caea0 pc=0x2321178\\nnet/http.send(0x170ac699e8c0, {0x59a1ce0, 0x8175400}, {0x170ac69cb138?, 0x1f3bac6?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69cb0b0 sp=0x170ac69caec0 pc=0x22ad9ab\\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac699e8c0, {0x170ac69cb1e8?, 0x1fdd787?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69cb148 sp=0x170ac69cb0b0 pc=0x22ad1d8\\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac699e8c0)\\n\\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69cb338 sp=0x170ac69cb148 pc=0x22af817\\nnet/http.(*Client).Do(...)\\n\\t/usr/lib/golang/src/net/http/client.go:592\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e410}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3d80}, 0x170ac69a0300, {0x170ac65d9b00, 0x114})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69cb630 sp=0x170ac69cb338 pc=0x38de6bd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e410}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x170ac69a02a0, 0x0, {0x0, ...}, ...})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69cb888 sp=0x170ac69cb630 pc=0x38ddecd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e410?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de3a48?}, 0x3?, 0x2022a4d?)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69cb960 sp=0x170ac69cb888 pc=0x38dd40b\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194620?}, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x170ac5de3a48, 0x3, 0x4})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69cb9e0 sp=0x170ac69cb960 pc=0x38e51e2\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194620}, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x0?, 0x2?, 0x2700000000000000?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69cba78 sp=0x170ac69cb9e0 pc=0x38e4eaf\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac65f81c0}, 0x170ac65fa000)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69cbd60 sp=0x170ac69cba78 pc=0x391b5c5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac65f81c0?}, 0x0?)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69cbe88 sp=0x170ac69cbd60 pc=0x3929c16\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69cbf78 sp=0x170ac69cbe88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69cbfe0 sp=0x170ac69cbf78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69cbfe8 sp=0x170ac69cbfe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 110 gp=0x170ac61d4b40 m=nil [select]:\\nruntime.gopark(0x170ac6927f08?, 0x6?, 0x40?, 0x4b?, 0x170ac6927da4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6927c08 sp=0x170ac6927be8 pc=0x1f38d2e\\nruntime.selectgo(0x170ac6927f08, 0x170ac6927d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6927d38 sp=0x170ac6927c08 pc=0x1f152a5\\nnet/http.(*http2clientStream).writeRequest(0x170ac61a0f00, 0x170ac619fb80, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6927f90 sp=0x170ac6927d38 pc=0x22da6a8\\nnet/http.(*http2clientStream).doRequest(0x170ac61a0f00, 0x0?, 0x170ac66e2798?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6927fb8 sp=0x170ac6927f90 pc=0x22d99d8\\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6927fe0 sp=0x170ac6927fb8 pc=0x22d999f\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6927fe8 sp=0x170ac6927fe0 pc=0x1f40801\\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 104\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\\n\\ngoroutine 111 gp=0x170ac61d4d20 m=nil [select]:\\nruntime.gopark(0x170ac658df08?, 0x6?, 0x0?, 0x7?, 0x170ac658dda4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac658dc08 sp=0x170ac658dbe8 pc=0x1f38d2e\\nruntime.selectgo(0x170ac658df08, 0x170ac658dd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac658dd38 sp=0x170ac658dc08 pc=0x1f152a5\\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1080, 0x170ac699e000, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac658df90 sp=0x170ac658dd38 pc=0x22da6a8\\nnet/http.(*http2clientStream).doRequest(0x170ac61a1080, 0x0?, 0x170ac66e2798?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac658dfb8 sp=0x170ac658df90 pc=0x22d99d8\\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac658dfe0 sp=0x170ac658dfb8 pc=0x22d999f\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac658dfe8 sp=0x170ac658dfe0 pc=0x1f40801\\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 105\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\\n\\ngoroutine 112 gp=0x170ac61d4f00 m=nil [select]:\\nruntime.gopark(0x170ac6589f08?, 0x6?, 0x0?, 0x7?, 0x170ac6589da4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6589c08 sp=0x170ac6589be8 pc=0x1f38d2e\\nruntime.selectgo(0x170ac6589f08, 0x170ac6589d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6589d38 sp=0x170ac6589c08 pc=0x1f152a5\\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1200, 0x170ac699e280, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6589f90 sp=0x170ac6589d38 pc=0x22da6a8\\nnet/http.(*http2clientStream).doRequest(0x170ac61a1200, 0x0?, 0x170ac66e1f98?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6589fb8 sp=0x170ac6589f90 pc=0x22d99d8\\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6589fe0 sp=0x170ac6589fb8 pc=0x22d999f\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6589fe8 sp=0x170ac6589fe0 pc=0x1f40801\\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 106\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\\n\\ngoroutine 113 gp=0x170ac61d50e0 m=nil [select]:\\nruntime.gopark(0x170ac658ff08?, 0x6?, 0x0?, 0x7?, 0x170ac658fda4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac658fc08 sp=0x170ac658fbe8 pc=0x1f38d2e\\nruntime.selectgo(0x170ac658ff08, 0x170ac658fd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac658fd38 sp=0x170ac658fc08 pc=0x1f152a5\\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1380, 0x170ac699e780, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac658ff90 sp=0x170ac658fd38 pc=0x22da6a8\\nnet/http.(*http2clientStream).doRequest(0x170ac61a1380, 0x0?, 0x170ac66e1f98?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac658ffb8 sp=0x170ac658ff90 pc=0x22d99d8\\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac658ffe0 sp=0x170ac658ffb8 pc=0x22d999f\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac658ffe8 sp=0x170ac658ffe0 pc=0x1f40801\\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 108\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\\n\\ngoroutine 114 gp=0x170ac61d52c0 m=nil [select]:\\nruntime.gopark(0x170ac6923f08?, 0x6?, 0x0?, 0x7?, 0x170ac6923da4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6923c08 sp=0x170ac6923be8 pc=0x1f38d2e\\nruntime.selectgo(0x170ac6923f08, 0x170ac6923d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6923d38 sp=0x170ac6923c08 pc=0x1f152a5\\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1500, 0x170ac699ea00, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6923f90 sp=0x170ac6923d38 pc=0x22da6a8\\nnet/http.(*http2clientStream).doRequest(0x170ac61a1500, 0x0?, 0x170ac66e1f98?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6923fb8 sp=0x170ac6923f90 pc=0x22d99d8\\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6923fe0 sp=0x170ac6923fb8 pc=0x22d999f\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6923fe8 sp=0x170ac6923fe0 pc=0x1f40801\\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 109\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\\n\\ngoroutine 115 gp=0x170ac61d54a0 m=nil [select]:\\nruntime.gopark(0x170ac6925f08?, 0x6?, 0x38?, 0x5b?, 0x170ac6925da4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6925c08 sp=0x170ac6925be8 pc=0x1f38d2e\\nruntime.selectgo(0x170ac6925f08, 0x170ac6925d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6925d38 sp=0x170ac6925c08 pc=0x1f152a5\\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1680, 0x170ac699e500, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6925f90 sp=0x170ac6925d38 pc=0x22da6a8\\nnet/http.(*http2clientStream).doRequest(0x170ac61a1680, 0x0?, 0x170ac66e1f98?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6925fb8 sp=0x170ac6925f90 pc=0x22d99d8\\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6925fe0 sp=0x170ac6925fb8 pc=0x22d999f\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6925fe8 sp=0x170ac6925fe0 pc=0x1f40801\\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 107\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\\n\\nrax    0x0\\nrbx    0x7f7eabfff640\\nrcx    0x7f7ef9baa94c\\nrdx    0x6\\nrdi    0x10a2e\\nrsi    0x10a30\\nrbp    0x10a30\\nrsp    0x7f7eabffe790\\nr8     0x7f7eabffe860\\nr9     0x7f7ef9cd04e0\\nr10    0x8\\nr11    0x246\\nr12    0x6\\nr13    0x7f7eabffeb60\\nr14    0x170ac5c1f0e0\\nr15    0xffffffffffffffff\\nrip    0x7f7ef9baa94c\\nrflags 0x246\\ncs     0x33\\nfs     0x0\\ngs     0x0\\n\\n  query-infra-both: command failed: exit status 2\\noutput: runtime/cgo: pthread_create failed: Resource temporarily unavailable\\nSIGABRT: abort\\nPC=0x7f5e9c16994c m=2 sigcode=18446744073709551610\\n\\ngoroutine 0 gp=0x280029a48b40 m=2 mp=0x280029abb008 [idle]:\\nruntime: g 0 gp=0x280029a48b40: unknown pc 0x7f5e9c16994c\\nstack: frame={sp:0x7f5e554a7740, fp:0x0} stack=[0x7f5e54ca81e0,0x7f5e554a7de0)\\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\\n00007f5e554a7640: 00000000 01eb64e5  00000000 00000000  .d..............\\n                  ^ \u003cinternal/runtime/syscall/linux.EpollWait+0x45\u003e\\n00007f5e554a7650: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7660: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7670: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7680: 00007f5e 554a76d4  00007f5e 554a7d18  .vJU^....}JU^...\\n00007f5e554a7690: 00000000 01efa894  00000000 00000000  ................\\n                  ^ \u003cruntime.netpoll+0xd4\u003e\\n00007f5e554a76a0: 00007f5e 554a76d4  00000000 00000000  .vJU^...........\\n00007f5e554a76b0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a76c0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a76d0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a76e0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a76f0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7700: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7710: 00000000 00000000  00007f5e 9c176fa9  .........o..^...\\n00007f5e554a7720: 00000000 00000000  00000000 00000011  ................\\n00007f5e554a7730: 00000000 00000130  00007f5e 9c16993e  0.......\u003e...^...\\n00007f5e554a7740: 00000000 00000120  00000000 00000000   ...............\\n                  ^ SP\\n00007f5e554a7750: 00000000 00000013  00000000 00000000  ................\\n00007f5e554a7760: 00000000 00000000  00007f5e 9c1eccff  ............^...\\n00007f5e554a7770: 00007f5e 327fc910  00007f5e 9c1eccb1  ...2^.......^...\\n00007f5e554a7780: 00007f5e 327fc640  00007f5e 554a7a50  @..2^...PzJU^...\\n00007f5e554a7790: 00007f5e 554a78be  00007f5e 554a78bf  .xJU^....xJU^...\\n00007f5e554a77a0: 00007f5e 327fc640  00007f5e 9c1677f5  @..2^....w..^...\\n00007f5e554a77b0: 00007f5e 50000020  00000000 00000120   ..P^... .......\\n00007f5e554a77c0: 00000000 003d0f00  5a23f32c fe4c8400  ..=.......L.,.#Z\\n00007f5e554a77d0: 00007f5e 554a8640  00000000 00000006  @.JU^...........\\n00007f5e554a77e0: 00000000 0bac42a0  00007f5e 554a7b10  .B.......{JU^...\\n00007f5e554a77f0: 00002800 29a48b40  00007f5e 9c11c646  @..).(..F...^...\\n00007f5e554a7800: 00007f5e 9c2d9e90  00007f5e 9c1067f3  ..-.^....g..^...\\n00007f5e554a7810: 00000000 00000020  5a23f32c fe4c8400   .........L.,.#Z\\n00007f5e554a7820: 00007f5e 31ffc000  00007f5e 9c1dbac4  ...1^.......^...\\n00007f5e554a7830: 00000000 00000000  00000000 0000000d  ................\\nruntime: g 0 gp=0x280029a48b40: unknown pc 0x7f5e9c16994c\\nstack: frame={sp:0x7f5e554a7740, fp:0x0} stack=[0x7f5e54ca81e0,0x7f5e554a7de0)\\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\\n00007f5e554a7640: 00000000 01eb64e5  00000000 00000000  .d..............\\n                  ^ \u003cinternal/runtime/syscall/linux.EpollWait+0x45\u003e\\n00007f5e554a7650: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7660: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7670: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7680: 00007f5e 554a76d4  00007f5e 554a7d18  .vJU^....}JU^...\\n00007f5e554a7690: 00000000 01efa894  00000000 00000000  ................\\n                  ^ \u003cruntime.netpoll+0xd4\u003e\\n00007f5e554a76a0: 00007f5e 554a76d4  00000000 00000000  .vJU^...........\\n00007f5e554a76b0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a76c0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a76d0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a76e0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a76f0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7700: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7710: 00000000 00000000  00007f5e 9c176fa9  .........o..^...\\n00007f5e554a7720: 00000000 00000000  00000000 00000011  ................\\n00007f5e554a7730: 00000000 00000130  00007f5e 9c16993e  0.......\u003e...^...\\n00007f5e554a7740: 00000000 00000120  00000000 00000000   ...............\\n                  ^ SP\\n00007f5e554a7750: 00000000 00000013  00000000 00000000  ................\\n00007f5e554a7760: 00000000 00000000  00007f5e 9c1eccff  ............^...\\n00007f5e554a7770: 00007f5e 327fc910  00007f5e 9c1eccb1  ...2^.......^...\\n00007f5e554a7780: 00007f5e 327fc640  00007f5e 554a7a50  @..2^...PzJU^...\\n00007f5e554a7790: 00007f5e 554a78be  00007f5e 554a78bf  .xJU^....xJU^...\\n00007f5e554a77a0: 00007f5e 327fc640  00007f5e 9c1677f5  @..2^....w..^...\\n00007f5e554a77b0: 00007f5e 50000020  00000000 00000120   ..P^... .......\\n00007f5e554a77c0: 00000000 003d0f00  5a23f32c fe4c8400  ..=.......L.,.#Z\\n00007f5e554a77d0: 00007f5e 554a8640  00000000 00000006  @.JU^...........\\n00007f5e554a77e0: 00000000 0bac42a0  00007f5e 554a7b10  .B.......{JU^...\\n00007f5e554a77f0: 00002800 29a48b40  00007f5e 9c11c646  @..).(..F...^...\\n00007f5e554a7800: 00007f5e 9c2d9e90  00007f5e 9c1067f3  ..-.^....g..^...\\n00007f5e554a7810: 00000000 00000020  5a23f32c fe4c8400   .........L.,.#Z\\n00007f5e554a7820: 00007f5e 31ffc000  00007f5e 9c1dbac4  ...1^.......^...\\n00007f5e554a7830: 00000000 00000000  00000000 0000000d  ................\\n\\ngoroutine 1 gp=0x280029a481e0 m=nil [sync.WaitGroup.Wait]:\\nruntime.gopark(0x28002a5927b8?, 0x2022a4d?, 0x30?, 0xe9?, 0x28002a5927e8?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a5927a0 sp=0x28002a592780 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x280029ae34d0, 0x0, 0x1, 0x0, 0x19)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a592808 sp=0x28002a5927a0 pc=0x1f162b2\\nsync.runtime_SemacquireWaitGroup(0x28002a83b578?, 0xe0?)\\n\\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x28002a592840 sp=0x28002a592808 pc=0x1f3a66e\\nsync.(*WaitGroup).Wait(0x280029ae34c8)\\n\\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x28002a592868 sp=0x28002a592840 pc=0x1f4f445\\ngolang.org/x/sync/errgroup.(*Group).Wait(0x280029ae34c0)\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x28002a592888 sp=0x28002a592868 pc=0x362507e\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0}, {0x28002a977b50, 0x1, 0x1})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x28002a592948 sp=0x28002a592888 pc=0x3927e05\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).gatherInfraLogs(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:587 +0x867 fp=0x28002a592d30 sp=0x28002a592948 pc=0x3927727\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:333 +0x533 fp=0x28002a593908 sp=0x28002a592d30 pc=0x3921bf3\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedInfraQueryOptions).RunInfra(0x28002a11e1e0, {0x59f0b08, 0x28002a1873b0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:83 +0x71f fp=0x28002a593b30 sp=0x28002a593908 pc=0x393887f\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawInfraQueryOptions).Run(0x0?, {0x59f0b08, 0x28002a1873b0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_options.go:109 +0x54 fp=0x28002a593b58 sp=0x28002a593b30 pc=0x3939134\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryInfraCommand.func1(0x28002a53d508?, {0x280029ca3b00?, 0x4?, 0x553b0e2?})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:45 +0x28 fp=0x28002a593b80 sp=0x28002a593b58 pc=0x3938128\\ngithub.com/spf13/cobra.(*Command).execute(0x28002a53d508, {0x280029ab8120, 0xc, 0xc})\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x28002a593d28 sp=0x28002a593b80 pc=0x212e614\\ngithub.com/spf13/cobra.(*Command).ExecuteC(0x280029ec9b08)\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x28002a593e18 sp=0x28002a593d28 pc=0x212efa5\\ngithub.com/spf13/cobra.(*Command).Execute(...)\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071\\ngithub.com/spf13/cobra.(*Command).ExecuteContext(...)\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064\\nmain.main()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x28002a593f48 sp=0x28002a593e18 pc=0x47ddd6a\\nruntime.main()\\n\\t/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x28002a593fe0 sp=0x28002a593f48 pc=0x1f01975\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a593fe8 sp=0x28002a593fe0 pc=0x1f40801\\n\\ngoroutine 2 gp=0x280029a48d20 m=nil [force gc (idle)]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b08fa8 sp=0x280029b08f88 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.forcegchelper()\\n\\t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x280029b08fe0 sp=0x280029b08fa8 pc=0x1f01c93\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b08fe8 sp=0x280029b08fe0 pc=0x1f40801\\ncreated by runtime.init.7 in goroutine 1\\n\\t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\\n\\ngoroutine 3 gp=0x280029a492c0 m=nil [GC sweep wait]:\\nruntime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b09788 sp=0x280029b09768 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.bgsweep(0x280029b30000)\\n\\t/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x280029b097c8 sp=0x280029b09788 pc=0x1ee9f11\\nruntime.gcenable.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x280029b097e0 sp=0x280029b097c8 pc=0x1edadd7\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b097e8 sp=0x280029b097e0 pc=0x1f40801\\ncreated by runtime.gcenable in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\\n\\ngoroutine 4 gp=0x280029a494a0 m=nil [GC scavenge wait]:\\nruntime.gopark(0x10000?, 0x5981cd8?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b09f78 sp=0x280029b09f58 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.(*scavengerState).park(0x82979c0)\\n\\t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x280029b09fa8 sp=0x280029b09f78 pc=0x1ee77e9\\nruntime.bgscavenge(0x280029b30000)\\n\\t/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x280029b09fc8 sp=0x280029b09fa8 pc=0x1ee7d99\\nruntime.gcenable.gowrap2()\\n\\t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x280029b09fe0 sp=0x280029b09fc8 pc=0x1edad97\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b09fe8 sp=0x280029b09fe0 pc=0x1f40801\\ncreated by runtime.gcenable in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\\n\\ngoroutine 5 gp=0x280029a49a40 m=nil [GOMAXPROCS updater (idle)]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0a788 sp=0x280029b0a768 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.updateMaxProcsGoroutine()\\n\\t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x280029b0a7e0 sp=0x280029b0a788 pc=0x1f10267\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0a7e8 sp=0x280029b0a7e0 pc=0x1f40801\\ncreated by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\\n\\t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\\n\\ngoroutine 6 gp=0x280029b481e0 m=nil [finalizer wait]:\\nruntime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x280029b08601?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b08620 sp=0x280029b08600 pc=0x1f38d2e\\nruntime.runFinalizers()\\n\\t/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x280029b087e0 sp=0x280029b08620 pc=0x1ed9cc7\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b087e8 sp=0x280029b087e0 pc=0x1f40801\\ncreated by runtime.createfing in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d\\n\\ngoroutine 7 gp=0x280029d3fc20 m=nil [cleanup wait]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0af68 sp=0x280029b0af48 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.(*cleanupQueue).dequeue(0x82980e0)\\n\\t/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x280029b0afa0 sp=0x280029b0af68 pc=0x1ed6074\\nruntime.runCleanups()\\n\\t/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x280029b0afe0 sp=0x280029b0afa0 pc=0x1ed66e5\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0afe8 sp=0x280029b0afe0 pc=0x1f40801\\ncreated by runtime.(*cleanupQueue).createGs in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5\\n\\ngoroutine 8 gp=0x280029ee6000 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125c86ae32d7?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0b740 sp=0x280029b0b720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x280029ad7ab0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x280029b0b7c8 sp=0x280029b0b740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b0b7e0 sp=0x280029b0b7c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0b7e8 sp=0x280029b0b7e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 18 gp=0x280029b843c0 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125c662e5759?, 0x3?, 0xc2?, 0x10?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b04740 sp=0x280029b04720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x280029ad7ab0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x280029b047c8 sp=0x280029b04740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b047e0 sp=0x280029b047c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b047e8 sp=0x280029b047e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 9 gp=0x280029ee61e0 m=nil [semacquire]:\\nruntime.gopark(0x6e2b22?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029b0bc96?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0be38 sp=0x280029b0be18 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029b0bea0 sp=0x280029b0be38 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029b0bf40 sp=0x280029b0bea0 pc=0x1edbb12\\nruntime.gcBgMarkWorker(0x280029ad7ab0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029b0bfc8 sp=0x280029b0bf40 pc=0x1edda88\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b0bfe0 sp=0x280029b0bfc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0bfe8 sp=0x280029b0bfe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 10 gp=0x280029ee63c0 m=nil [semacquire]:\\nruntime.gopark(0x1ec6f5e?, 0x1f3ec12?, 0x80?, 0x51?, 0x1ec70df?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eec638 sp=0x280029eec618 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eec6a0 sp=0x280029eec638 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eec740 sp=0x280029eec6a0 pc=0x1edbb12\\nruntime.gcBgMarkWorker(0x280029ad7ab0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eec7c8 sp=0x280029eec740 pc=0x1edda88\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eec7e0 sp=0x280029eec7c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eec7e8 sp=0x280029eec7e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 11 gp=0x280029ee65a0 m=nil [semacquire]:\\nruntime.gopark(0x54b522?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029eecc96?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eece38 sp=0x280029eece18 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eecea0 sp=0x280029eece38 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eecf40 sp=0x280029eecea0 pc=0x1edbb12\\nruntime.gcBgMarkWorker(0x280029ad7ab0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eecfc8 sp=0x280029eecf40 pc=0x1edda88\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eecfe0 sp=0x280029eecfc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eecfe8 sp=0x280029eecfe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 34 gp=0x280029f04000 m=nil [semacquire]:\\nruntime.gopark(0x2d7122?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029ee8496?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029ee8638 sp=0x280029ee8618 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029ee86a0 sp=0x280029ee8638 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029ee8740 sp=0x280029ee86a0 pc=0x1edbb12\\nruntime.gcBgMarkWorker(0x280029ad7ab0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029ee87c8 sp=0x280029ee8740 pc=0x1edda88\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029ee87e0 sp=0x280029ee87c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029ee87e8 sp=0x280029ee87e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 19 gp=0x280029b845a0 m=4 mp=0x280029b34008 [GC mark termination]:\\nruntime.systemstack_switch()\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x280029b04ea0 sp=0x280029b04e90 pc=0x1f3ebc8\\nruntime.forEachP(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:2112\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x280029b04f40 sp=0x280029b04ea0 pc=0x1edbc34\\nruntime.gcBgMarkWorker(0x280029ad7ab0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029b04fc8 sp=0x280029b04f40 pc=0x1edda88\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b04fe0 sp=0x280029b04fc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b04fe8 sp=0x280029b04fe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 12 gp=0x280029ee6780 m=nil [semacquire]:\\nruntime.gopark(0x3f9e22?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029eed496?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eed638 sp=0x280029eed618 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eed6a0 sp=0x280029eed638 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eed740 sp=0x280029eed6a0 pc=0x1edbb12\\nruntime.gcBgMarkWorker(0x280029ad7ab0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eed7c8 sp=0x280029eed740 pc=0x1edda88\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eed7e0 sp=0x280029eed7c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eed7e8 sp=0x280029eed7e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 35 gp=0x280029ee74a0 m=nil [select, locked to thread]:\\nruntime.gopark(0x280029eef7a8?, 0x2?, 0x1?, 0x74?, 0x280029eef794?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eef628 sp=0x280029eef608 pc=0x1f38d2e\\nruntime.selectgo(0x280029eef7a8, 0x280029eef790, 0x0?, 0x0, 0x0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x280029eef758 sp=0x280029eef628 pc=0x1f152a5\\nruntime.ensureSigM.func1()\\n\\t/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x280029eef7e0 sp=0x280029eef758 pc=0x1f328e8\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eef7e8 sp=0x280029eef7e0 pc=0x1f40801\\ncreated by runtime.ensureSigM in goroutine 1\\n\\t/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5\\n\\ngoroutine 50 gp=0x280029f84960 m=9 mp=0x28002a000008 [syscall]:\\nruntime.notetsleepg(0x82bdea0, 0xffffffffffffffff)\\n\\t/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x280029ee9fa0 sp=0x280029ee9f78 pc=0x1ece509\\nos/signal.signal_recv()\\n\\t/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x280029ee9fc0 sp=0x280029ee9fa0 pc=0x1f3b298\\nos/signal.loop()\\n\\t/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x280029ee9fe0 sp=0x280029ee9fc0 pc=0x2030553\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029ee9fe8 sp=0x280029ee9fe0 pc=0x1f40801\\ncreated by os/signal.Notify.func1.1 in goroutine 1\\n\\t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\\n\\ngoroutine 51 gp=0x280029f84b40 m=nil [select]:\\nruntime.gopark(0x280029eea798?, 0x2?, 0x80?, 0xa3?, 0x280029eea78c?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eea628 sp=0x280029eea608 pc=0x1f38d2e\\nruntime.selectgo(0x280029eea798, 0x280029eea788, 0x0?, 0x0, 0x0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x280029eea758 sp=0x280029eea628 pc=0x1f152a5\\nos/signal.NotifyContext.func1()\\n\\t/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x280029eea7e0 sp=0x280029eea758 pc=0x202ff9c\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eea7e8 sp=0x280029eea7e0 pc=0x1f40801\\ncreated by os/signal.NotifyContext in goroutine 1\\n\\t/usr/lib/golang/src/os/signal/signal.go:291 +0x158\\n\\ngoroutine 137 gp=0x28002a60a000 m=nil [sync.WaitGroup.Wait]:\\nruntime.gopark(0x28002a603e00?, 0x1ed0765?, 0x80?, 0xea?, 0x7f5e54234a40?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a603da0 sp=0x28002a603d80 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x280029ae3b10, 0x0, 0x1, 0x0, 0x19)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a603e08 sp=0x28002a603da0 pc=0x1f162b2\\nsync.runtime_SemacquireWaitGroup(0x28002a83b590?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x28002a603e40 sp=0x28002a603e08 pc=0x1f3a66e\\nsync.(*WaitGroup).Wait(0x280029ae3b08)\\n\\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x28002a603e68 sp=0x28002a603e40 pc=0x1f4f445\\ngolang.org/x/sync/errgroup.(*Group).Wait(0x280029ae3b00)\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x28002a603e88 sp=0x28002a603e68 pc=0x362507e\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x280029d80e80, {0x59f0b40, 0x280029beca00}, {0x28002a977b50, 0x1, 0x38d788c?}, 0x28002a830c40)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x28002a603f00 sp=0x28002a603e88 pc=0x392976a\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x28002a603f78 sp=0x28002a603f00 pc=0x392808e\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a603fe0 sp=0x28002a603f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a603fe8 sp=0x28002a603fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 38 gp=0x28002a60b0e0 m=nil [IO wait]:\\nruntime.gopark(0x28002a7c22c0?, 0x280029aee7f8?, 0x40?, 0x77?, 0xb?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b197b8 sp=0x280029b19798 pc=0x1f38d2e\\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x280029b197f0 sp=0x280029b197b8 pc=0x1ef9e57\\ninternal/poll.runtime_pollWait(0x7f5e554b6c00, 0x72)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x280029b19810 sp=0x280029b197f0 pc=0x1f375c5\\ninternal/poll.(*pollDesc).wait(0x280029bf6400?, 0x28002a90c000?, 0x0)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x280029b19838 sp=0x280029b19810 pc=0x1fbd167\\ninternal/poll.(*pollDesc).waitRead(...)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\\ninternal/poll.(*FD).Read(0x280029bf6400, {0x28002a90c000, 0x3500, 0x3500})\\n\\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x280029b198d0 sp=0x280029b19838 pc=0x1fbe4ce\\nnet.(*netFD).Read(0x280029bf6400, {0x28002a90c000?, 0x28002a90c000?, 0x5?})\\n\\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x280029b19918 sp=0x280029b198d0 pc=0x20798c5\\nnet.(*conn).Read(0x280029a46378, {0x28002a90c000?, 0x7f5e554a9828?, 0x7f5e9c0b1e00?})\\n\\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x280029b19960 sp=0x280029b19918 pc=0x208b7a5\\ncrypto/tls.(*atLeastReader).Read(0x28002aa26ba0, {0x28002a90c000?, 0x28002a0d8658?, 0x1efcc28?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x280029b199a8 sp=0x280029b19960 pc=0x2222ddb\\nbytes.(*Buffer).ReadFrom(0x28002a0d8628, {0x59a3680, 0x28002aa26ba0})\\n\\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x280029b19a00 sp=0x280029b199a8 pc=0x1fe9258\\ncrypto/tls.(*Conn).readFromUntil(0x28002a0d8388, {0x59a32c0, 0x280029a46378}, 0x222764c?)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x280029b19a38 sp=0x280029b19a00 pc=0x2222fbe\\ncrypto/tls.(*Conn).readRecordOrCCS(0x28002a0d8388, 0x0)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x280029b19c18 sp=0x280029b19a38 pc=0x222009b\\ncrypto/tls.(*Conn).readRecord(...)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:588\\ncrypto/tls.(*Conn).Read(0x28002a0d8388, {0x280029fed000, 0x1000, 0x22c34e0?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x280029b19c80 sp=0x280029b19c18 pc=0x22267c5\\nbufio.(*Reader).Read(0x28002a7b89c0, {0x28002a7c8584, 0x9, 0x22e100e?})\\n\\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0x280029b19cb8 sp=0x280029b19c80 pc=0x20a71f7\\nio.ReadAtLeast({0x59a1260, 0x28002a7b89c0}, {0x28002a7c8584, 0x9, 0x9}, 0x9)\\n\\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0x280029b19d00 sp=0x280029b19cb8 pc=0x1fb17ee\\nio.ReadFull(...)\\n\\t/usr/lib/golang/src/io/io.go:354\\nnet/http.http2readFrameHeader({0x28002a7c8584, 0x9, 0x280000000015?}, {0x59a1260?, 0x28002a7b89c0?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:1805 +0x65 fp=0x280029b19d50 sp=0x280029b19d00 pc=0x22b9e25\\nnet/http.(*http2Framer).ReadFrameHeader(0x28002a7c8540)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:2071 +0x6b fp=0x280029b19dd8 sp=0x280029b19d50 pc=0x22ba9eb\\nnet/http.(*http2Framer).ReadFrame(0x28002a7c8540)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:2130 +0x18 fp=0x280029b19e18 sp=0x280029b19dd8 pc=0x22bb1b8\\nnet/http.(*http2clientConnReadLoop).run(0x280029b19fa8)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9550 +0xca fp=0x280029b19f60 sp=0x280029b19e18 pc=0x22de14a\\nnet/http.(*http2ClientConn).readLoop(0x280029bc01c0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x280029b19fc8 sp=0x280029b19f60 pc=0x22dd532\\nnet/http.(*http2Transport).newClientConn.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x280029b19fe0 sp=0x280029b19fc8 pc=0x22d7057\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b19fe8 sp=0x280029b19fe0 pc=0x1f40801\\ncreated by net/http.(*http2Transport).newClientConn in goroutine 37\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5\\n\\ngoroutine 69 gp=0x28002a60b4a0 m=nil [IO wait]:\\nruntime.gopark(0x28002a50c560?, 0x1edb176?, 0x80?, 0xed?, 0xb?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a606808 sp=0x28002a6067e8 pc=0x1f38d2e\\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x28002a606840 sp=0x28002a606808 pc=0x1ef9e57\\ninternal/poll.runtime_pollWait(0x7f5e554b6a00, 0x72)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x28002a606860 sp=0x28002a606840 pc=0x1f375c5\\ninternal/poll.(*pollDesc).wait(0x280029bf6c80?, 0x280029a5a000?, 0x0)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x28002a606888 sp=0x28002a606860 pc=0x1fbd167\\ninternal/poll.(*pollDesc).waitRead(...)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\\ninternal/poll.(*FD).Read(0x280029bf6c80, {0x280029a5a000, 0x2500, 0x2500})\\n\\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x28002a606920 sp=0x28002a606888 pc=0x1fbe4ce\\nnet.(*netFD).Read(0x280029bf6c80, {0x280029a5a000?, 0x28002a60690a?, 0x1edb176?})\\n\\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x28002a606968 sp=0x28002a606920 pc=0x20798c5\\nnet.(*conn).Read(0x280029a463c0, {0x280029a5a000?, 0x7f5e5423eae0?, 0x7f5e9c09dde0?})\\n\\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x28002a6069b0 sp=0x28002a606968 pc=0x208b7a5\\ncrypto/tls.(*atLeastReader).Read(0x28002a628318, {0x280029a5a000?, 0x28002a606a98?, 0x1efcc28?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x28002a6069f8 sp=0x28002a6069b0 pc=0x2222ddb\\nbytes.(*Buffer).ReadFrom(0x280029ba9ea8, {0x59a3680, 0x28002a628318})\\n\\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x28002a606a50 sp=0x28002a6069f8 pc=0x1fe9258\\ncrypto/tls.(*Conn).readFromUntil(0x280029ba9c08, {0x59a32c0, 0x280029a463c0}, 0x222764c?)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x28002a606a88 sp=0x28002a606a50 pc=0x2222fbe\\ncrypto/tls.(*Conn).readRecordOrCCS(0x280029ba9c08, 0x0)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x28002a606c68 sp=0x28002a606a88 pc=0x222009b\\ncrypto/tls.(*Conn).readRecord(...)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:588\\ncrypto/tls.(*Conn).Read(0x280029ba9c08, {0x28002a24a000, 0x1000, 0x4e55540?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x28002a606cd0 sp=0x28002a606c68 pc=0x22267c5\\nnet/http.(*persistConn).Read(0x28002a0ad400, {0x28002a24a000?, 0x59a0ea0?, 0x81260e0?})\\n\\t/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x28002a606d30 sp=0x28002a606cd0 pc=0x2319d07\\nbufio.(*Reader).fill(0x28002a8130e0)\\n\\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x28002a606d68 sp=0x28002a606d30 pc=0x20a6c63\\nbufio.(*Reader).Peek(0x28002a8130e0, 0x1)\\n\\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x28002a606d80 sp=0x28002a606d68 pc=0x20a6df3\\nnet/http.(*persistConn).readLoop(0x28002a0ad400)\\n\\t/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x28002a606fc8 sp=0x28002a606d80 pc=0x231a812\\nnet/http.(*Transport).dialConn.gowrap2()\\n\\t/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x28002a606fe0 sp=0x28002a606fc8 pc=0x2319197\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a606fe8 sp=0x28002a606fe0 pc=0x1f40801\\ncreated by net/http.(*Transport).dialConn in goroutine 55\\n\\t/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac\\n\\ngoroutine 70 gp=0x28002a60bc20 m=nil [select]:\\nruntime.gopark(0x28002a613f48?, 0x2?, 0x0?, 0xb6?, 0x28002a613ef4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a708d88 sp=0x28002a708d68 pc=0x1f38d2e\\nruntime.selectgo(0x28002a708f48, 0x28002a613ef0, 0x280029c93a80?, 0x0, 0x28002a4f7c80?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a708eb8 sp=0x28002a708d88 pc=0x1f152a5\\nnet/http.(*persistConn).writeLoop(0x28002a0ad400)\\n\\t/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x28002a708fc8 sp=0x28002a708eb8 pc=0x231c426\\nnet/http.(*Transport).dialConn.gowrap3()\\n\\t/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x28002a708fe0 sp=0x28002a708fc8 pc=0x2319157\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a708fe8 sp=0x28002a708fe0 pc=0x1f40801\\ncreated by net/http.(*Transport).dialConn in goroutine 55\\n\\t/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe\\n\\ngoroutine 138 gp=0x28002a84a780 m=nil [semacquire]:\\nruntime.gopark(0x1?, 0x1f78fa5?, 0xe0?, 0x75?, 0x2?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a2ca910 sp=0x28002a2ca8f0 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a2ca978 sp=0x28002a2ca910 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x28002a2caa18 sp=0x28002a2ca978 pc=0x1edbb12\\nruntime.gcAssistAlloc(0x28002a84a780)\\n\\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x28002a2caaa8 sp=0x28002a2caa18 pc=0x1ee06b6\\nruntime.deductAssistCredit(0x28002a2cac70?)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x28002a2caac8 sp=0x28002a2caaa8 pc=0x1ed10a6\\nruntime.mallocgc(0x10, 0x4af39e0, 0x1)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x28002a2caaf8 sp=0x28002a2caac8 pc=0x1f35e07\\nruntime.convTstring({0x28002a6a7728, 0x14})\\n\\t/usr/lib/golang/src/runtime/iface.go:431 +0x45 fp=0x28002a2cab20 sp=0x28002a2caaf8 pc=0x1f35865\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x280029beca00}, 0x28002a830cb0, 0x280029f636e0)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:297 +0x394 fp=0x28002a2caf48 sp=0x28002a2cab20 pc=0x3920674\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x28002a2caf78 sp=0x28002a2caf48 pc=0x392800c\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a2cafe0 sp=0x28002a2caf78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a2cafe8 sp=0x28002a2cafe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 139 gp=0x28002a84a960 m=nil [select]:\\nruntime.gopark(0x28002a4bdd00?, 0x2?, 0x1?, 0xc9?, 0x28002a4bdc74?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a4bdac8 sp=0x28002a4bdaa8 pc=0x1f38d2e\\nruntime.selectgo(0x28002a4bdd00, 0x28002a4bdc70, 0x28002a3e8a50?, 0x0, 0x280029f827c8?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a4bdbf8 sp=0x28002a4bdac8 pc=0x1f152a5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x49424d554554534c?, {0x59f0b40, 0x280029beca00}, 0x28002a830c40, 0x28002a830cb0)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x28002a4bdf10 sp=0x28002a4bdbf8 pc=0x392836e\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x28002a4bdf78 sp=0x28002a4bdf10 pc=0x3927f3d\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a4bdfe0 sp=0x28002a4bdf78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a4bdfe8 sp=0x28002a4bdfe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 140 gp=0x28002a84ab40 m=nil [semacquire]:\\nruntime.gopark(0x1f6aa32?, 0x28002a34b418?, 0xc5?, 0xbd?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a34b3c0 sp=0x28002a34b3a0 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a34b428 sp=0x28002a34b3c0 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x28002a34b4c8 sp=0x28002a34b428 pc=0x1edbb12\\nruntime.gcAssistAlloc(0x28002a84ab40)\\n\\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x28002a34b558 sp=0x28002a34b4c8 pc=0x1ee06b6\\nruntime.deductAssistCredit(0x20?)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x28002a34b578 sp=0x28002a34b558 pc=0x1ed10a6\\nruntime.mallocgc(0x1000, 0x4af32a0, 0x1)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x28002a34b5a8 sp=0x28002a34b578 pc=0x1f35e07\\nruntime.makeslice(0x28002a34b668?, 0x1f8?, 0x1ff?)\\n\\t/usr/lib/golang/src/runtime/slice.go:117 +0x49 fp=0x28002a34b5d0 sp=0x28002a34b5a8 pc=0x1f3b489\\nbufio.NewWriterSize(...)\\n\\t/usr/lib/golang/src/bufio/bufio.go:599\\nbufio.NewWriter(...)\\n\\t/usr/lib/golang/src/bufio/bufio.go:608\\nencoding/csv.NewWriter(...)\\n\\t/usr/lib/golang/src/encoding/csv/writer.go:42\\ngithub.com/Azure/azure-kusto-go/azkustodata/query.(*row).String(0x812ab90?)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/row_impl.go:100 +0x18e fp=0x28002a34b6e8 sp=0x28002a34b5d0 pc=0x38cf5ee\\nfmt.(*pp).handleMethods(0x28002a24c0d0, 0x2a34b8f0?)\\n\\t/usr/lib/golang/src/fmt/print.go:674 +0x2f4 fp=0x28002a34b8a8 sp=0x28002a34b6e8 pc=0x1fdd1d4\\nfmt.(*pp).printArg(0x28002a24c0d0, {0x549a3a0, 0x28002a7c7c80}, 0x76)\\n\\t/usr/lib/golang/src/fmt/print.go:757 +0x42e fp=0x28002a34b938 sp=0x28002a34b8a8 pc=0x1fdd9ee\\nfmt.(*pp).doPrintf(0x28002a24c0d0, {0x553a400, 0x2}, {0x28002a34bd08, 0x1, 0x1})\\n\\t/usr/lib/golang/src/fmt/print.go:1075 +0x3ec fp=0x28002a34ba20 sp=0x28002a34b938 pc=0x1fe04ec\\nfmt.Sprintf({0x553a400, 0x2}, {0x28002a34bd08, 0x1, 0x1})\\n\\t/usr/lib/golang/src/fmt/print.go:239 +0x53 fp=0x28002a34ba78 sp=0x28002a34ba20 pc=0x1fda6b3\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x280029e454e0, {0x59f0b40, 0x280029beca50}, {0x5a00300, 0x280029ae2040}, 0x28002a830c40)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:160 +0x8d8 fp=0x28002a34bd60 sp=0x28002a34ba78 pc=0x391bad8\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x2d20414320746f6f?, {0x59f0b40?, 0x280029beca50?}, {0x5a00300?, 0x280029ae2040?}, 0x43437a4b4349494d?)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x28002a34be88 sp=0x28002a34bd60 pc=0x3929c16\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x28002a34bf78 sp=0x28002a34be88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a34bfe0 sp=0x28002a34bf78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a34bfe8 sp=0x28002a34bfe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 137\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 142 gp=0x28002a84af00 m=nil [select]:\\nruntime.gopark(0x28002a333de0?, 0x2?, 0x8?, 0xb8?, 0x28002a333dbc?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a333c58 sp=0x28002a333c38 pc=0x1f38d2e\\nruntime.selectgo(0x28002a333de0, 0x28002a333db8, 0x280029afc008?, 0x1, 0x1f3665f?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a333d88 sp=0x28002a333c58 pc=0x1f152a5\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.(*iterativeDataset).sendTable(0x280029becf00, {0x5a19be0, 0x28002ab876c0})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:378 +0xf3 fp=0x28002a333e10 sp=0x28002a333d88 pc=0x38d8d33\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.handleDataTable(0x280029becf00, 0x28002a3b2b40)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:324 +0x2c6 fp=0x28002a333ec8 sp=0x28002a333e10 pc=0x38d8686\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x280029becf00)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:167 +0x148 fp=0x28002a333f30 sp=0x28002a333ec8 pc=0x38d7b48\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x280029becf00, 0x28002a977da0)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x28002a333fc0 sp=0x28002a333f30 pc=0x38d788c\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x28002a333fe0 sp=0x28002a333fc0 pc=0x38d751b\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a333fe8 sp=0x28002a333fe0 pc=0x1f40801\\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 140\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc\\n\\nrax    0x0\\nrbx    0x7f5e554a8640\\nrcx    0x7f5e9c16994c\\nrdx    0x6\\nrdi    0x10a66\\nrsi    0x10a67\\nrbp    0x10a67\\nrsp    0x7f5e554a7740\\nr8     0x7f5e554a7810\\nr9     0x7f5e9c28f4e0\\nr10    0x8\\nr11    0x246\\nr12    0x6\\nr13    0x7f5e554a7b10\\nr14    0x280029a48b40\\nr15    0xffffffffffffffff\\nrip    0x7f5e9c16994c\\nrflags 0x246\\ncs     0x33\\nfs     0x0\\ngs     0x0\\n\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:123]: Timed out after 651.749s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x48c3b8dec90\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[21:33:14.493]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104\"\n    }�[0m\n    �[37m[21:33:23.438]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:21.8572874Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\\\", ActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", SubActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \\u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[21:33:23.438]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 50/289 failed (KQL: let _startTime = datetime(2026-10-15T01:38:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T01:43:14.4931101Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-15T01:38:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T01:43:14.4931101Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[21:33:23.712]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/cosmosContent\"\n    }�[0m\n    �[37m[21:33:23.712]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[21:33:24.208]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 1498,\n      \"files\": 1\n    }�[0m\n    �[37m[21:33:24.208]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/cosmosContent\"\n    }�[0m\n    �[37m[21:33:24.217]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[21:33:40.606]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n      \"commits\": 793\n    }�[0m\n    �[37m[21:33:40.607]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect\"\n    }�[0m\n    �[37m[21:33:40.961]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n      ]\n    }�[0m\n    �[37m[21:33:41.088]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j8700672-mgmt-1\",\n        \"ci01-j8700672-svc\"\n      ]\n    }�[0m\n    �[37m[21:33:41.127]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespaces\": [\n        \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n      ]\n    }�[0m\n    �[37m[21:33:41.127]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespaceCount\": 4\n    }�[0m\n    �[37m[21:33:41.127]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:33:41.317]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:33:41.478]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:33:41.623]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n    }�[0m\n    �[37m[21:33:43.044]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:21.8572874Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\\\", ActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", SubActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \\u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\\\"}, IsPermanent:false})\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3888395104\"\n    }�[0m\n    �[37m[21:33:43.044]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:21.8572874Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\\\", ActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", SubActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \\u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\\\"}, IsPermanent:false})\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[21:33:43.407]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843\"\n    }�[0m\n    �[37m[21:33:49.648]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:47.6716888Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\\\", ActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", SubActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \\u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[21:33:49.648]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 47/289 failed (KQL: let _startTime = datetime(2026-10-15T01:23:43.4070857Z);\\nlet _endTime = datetime(2026-10-15T01:28:43.4070856Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[21:33:49.744]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/cosmosContent\"\n    }�[0m\n    �[37m[21:33:49.744]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[21:33:50.271]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 1498,\n      \"files\": 1\n    }�[0m\n    �[37m[21:33:50.271]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/cosmosContent\"\n    }�[0m\n    �[37m[21:33:50.281]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[21:34:08.647]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n      \"commits\": 793\n    }�[0m\n    �[37m[21:34:08.648]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect\"\n    }�[0m\n    �[37m[21:34:09.212]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n      ]\n    }�[0m\n    �[37m[21:34:09.309]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j8700672-mgmt-1\",\n        \"ci01-j8700672-svc\"\n      ]\n    }�[0m\n    �[37m[21:34:09.470]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespaces\": [\n        \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n      ]\n    }�[0m\n    �[37m[21:34:09.470]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespaceCount\": 4\n    }�[0m\n    �[37m[21:34:09.470]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:34:09.615]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:34:10.183]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:34:10.617]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n    }�[0m\n    �[37m[21:34:12.156]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:47.6716888Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\\\", ActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", SubActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \\u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\\\"}, IsPermanent:false})\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843\"\n    }�[0m\n    �[37m[21:34:12.157]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:33:47.6716888Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\\\", ActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", SubActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \\u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\\\"}, IsPermanent:false})\"\n    }�[0m\n    \n      query-skip-kubernetes-events: command failed: exit status 1\n    output: �[37m[21:34:12.559]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767\"\n    }�[0m\n    �[37m[21:34:18.220]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:16.3210435Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\\\", ActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", SubActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \\u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[21:34:18.221]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 47/289 failed (KQL: let _startTime = datetime(2026-10-15T01:24:12.5582990Z);\\nlet _endTime = datetime(2026-10-15T01:29:12.5582989Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[21:34:18.221]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/cosmosContent\"\n    }�[0m\n    �[37m[21:34:18.221]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[21:34:18.723]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 1498,\n      \"files\": 1\n    }�[0m\n    �[37m[21:34:18.724]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/cosmosContent\"\n    }�[0m\n    �[37m[21:34:18.734]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[21:34:35.437]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect\"\n    }�[0m\n    �[37m[21:34:35.845]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n      ]\n    }�[0m\n    �[37m[21:34:36.194]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j8700672-mgmt-1\",\n        \"ci01-j8700672-svc\"\n      ]\n    }�[0m\n    �[37m[21:34:36.313]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespaces\": [\n        \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n      ]\n    }�[0m\n    �[37m[21:34:36.313]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespaceCount\": 4\n    }�[0m\n    �[37m[21:34:36.313]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:34:36.544]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:34:36.863]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:34:37.093]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n    }�[0m\n    �[37m[21:34:38.438]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:16.3210435Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\\\", ActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", SubActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \\u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767\"\n    }�[0m\n    �[37m[21:34:38.438]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:16.3210435Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\\\", ActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", SubActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \\u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[21:34:38.772]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904\"\n    }�[0m\n    �[37m[21:34:46.539]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:46.3046137Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\\\", ActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", SubActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \\u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[21:34:46.539]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 45/289 failed (KQL: let _startTime = datetime(2026-10-15T01:14:38.7716298Z);\\nlet _endTime = datetime(2026-10-15T01:19:38.7716297Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[21:34:47.129]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/cosmosContent\"\n    }�[0m\n    �[37m[21:34:47.129]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[21:34:47.633]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 1498,\n      \"files\": 1\n    }�[0m\n    �[37m[21:34:47.633]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/cosmosContent\"\n    }�[0m\n    �[37m[21:34:47.642]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[21:34:59.291]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect\"\n    }�[0m\n    �[37m[21:34:59.911]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n      ]\n    }�[0m\n    �[37m[21:35:00.221]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j8700672-mgmt-1\",\n        \"ci01-j8700672-svc\"\n      ]\n    }�[0m\n    �[37m[21:35:00.360]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespaces\": [\n        \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n        \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n      ]\n    }�[0m\n    �[37m[21:35:00.360]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespaceCount\": 4\n    }�[0m\n    �[37m[21:35:00.360]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:35:00.702]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:35:01.111]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\"\n    }�[0m\n    �[37m[21:35:01.638]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8700672-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\"\n    }�[0m\n    �[37m[21:35:02.339]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:46.3046137Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\\\", ActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", SubActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \\u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904\"\n    }�[0m\n    �[37m[21:35:02.340]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:34:46.3046137Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\\\", ActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", SubActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \\u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n    }�[0m\n    \n      query-skip-hcp-and-events: command failed: exit status 2\n    output: �[37m[21:35:02.599]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-1950984811\"\n    }�[0m\n    runtime/cgo: pthread_create failed: Resource temporarily unavailable\n    SIGABRT: abort\n    PC=0x7f80df60594c m=4 sigcode=18446744073709551610\n    \n    goroutine 0 gp=0x14197f1334a0 m=4 mp=0x14197f21a008 [idle]:\n    runtime: g 0 gp=0x14197f1334a0: unknown pc 0x7f80df60594c\n    stack: frame={sp:0x7f808bffe790, fp:0x0} stack=[0x7f808b7ff1e0,0x7f808bffede0)\n                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n    00007f808bffe690: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6a0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6b0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6c0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6d0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6e0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6f0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe700: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe710: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe720: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe730: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe740: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe750: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe760: 00000000 00000000  00007f80 df612fa9  ........./a.....\n    00007f808bffe770: 00000000 00000000  00000000 00000011  ................\n    00007f808bffe780: 00000000 00000130  00007f80 df60593e  0.......\u003eY`.....\n    00007f808bffe790: 00000000 00000120  00000000 00000000   ...............\n                      ^ SP\n    00007f808bffe7a0: 00010000 00000013  00000000 00000000  ................\n    00007f808bffe7b0: 00000000 00000000  00007f80 df688cff  ..........h.....\n    00007f808bffe7c0: 00007f80 89ffb910  00007f80 df688cb1  ..........h.....\n    00007f808bffe7d0: 00007f80 89ffb640  00007f80 8bffeaa0  @...............\n    00007f808bffe7e0: 00007f80 8bffe90e  00007f80 8bffe90f  ................\n    00007f808bffe7f0: 00007f80 89ffb640  00007f80 df6037f5  @........7`.....\n    00007f808bffe800: 00007f80 84000020  00000000 00000120   ....... .......\n    00007f808bffe810: 00000000 003d0f00  701e1ebb c8982900  ..=......).....p\n    00007f808bffe820: 00007f80 8bfff640  00000000 00000006  @...............\n    00007f808bffe830: 00007f80 84000c90  00007f80 8bffeb60  ........`.......\n    00007f808bffe840: 00001419 7f1334a0  00007f80 df5b8646  .4......F.[.....\n    00007f808bffe850: 00007f80 df775e90  00007f80 df5a27f3  .^w......'Z.....\n    00007f808bffe860: 00000000 00000020  701e1ebb c8982900   ........).....p\n    00007f808bffe870: 00007f80 897fb000  00007f80 df677ac4  .........zg.....\n    00007f808bffe880: 00000000 00000000  00000000 0000000d  ................\n    runtime: g 0 gp=0x14197f1334a0: unknown pc 0x7f80df60594c\n    stack: frame={sp:0x7f808bffe790, fp:0x0} stack=[0x7f808b7ff1e0,0x7f808bffede0)\n                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n    00007f808bffe690: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6a0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6b0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6c0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6d0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6e0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe6f0: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe700: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe710: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe720: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe730: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe740: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe750: 00000000 00000000  00000000 00000000  ................\n    00007f808bffe760: 00000000 00000000  00007f80 df612fa9  ........./a.....\n    00007f808bffe770: 00000000 00000000  00000000 00000011  ................\n    00007f808bffe780: 00000000 00000130  00007f80 df60593e  0.......\u003eY`.....\n    00007f808bffe790: 00000000 00000120  00000000 00000000   ...............\n                      ^ SP\n    00007f808bffe7a0: 00010000 00000013  00000000 00000000  ................\n    00007f808bffe7b0: 00000000 00000000  00007f80 df688cff  ..........h.....\n    00007f808bffe7c0: 00007f80 89ffb910  00007f80 df688cb1  ..........h.....\n    00007f808bffe7d0: 00007f80 89ffb640  00007f80 8bffeaa0  @...............\n    00007f808bffe7e0: 00007f80 8bffe90e  00007f80 8bffe90f  ................\n    00007f808bffe7f0: 00007f80 89ffb640  00007f80 df6037f5  @........7`.....\n    00007f808bffe800: 00007f80 84000020  00000000 00000120   ....... .......\n    00007f808bffe810: 00000000 003d0f00  701e1ebb c8982900  ..=......).....p\n    00007f808bffe820: 00007f80 8bfff640  00000000 00000006  @...............\n    00007f808bffe830: 00007f80 84000c90  00007f80 8bffeb60  ........`.......\n    00007f808bffe840: 00001419 7f1334a0  00007f80 df5b8646  .4......F.[.....\n    00007f808bffe850: 00007f80 df775e90  00007f80 df5a27f3  .^w......'Z.....\n    00007f808bffe860: 00000000 00000020  701e1ebb c8982900   ........).....p\n    00007f808bffe870: 00007f80 897fb000  00007f80 df677ac4  .........zg.....\n    00007f808bffe880: 00000000 00000000  00000000 0000000d  ................\n    \n    goroutine 1 gp=0x14197f1321e0 m=nil [sync.WaitGroup.Wait]:\n    runtime.gopark(0x14197fe86c00?, 0x2022a4d?, 0x10?, 0x63?, 0x14197fe86c30?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197ff32be8 sp=0x14197ff32bc8 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x14197fc98610, 0x0, 0x1, 0x0, 0x19)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197ff32c50 sp=0x14197ff32be8 pc=0x1f162b2\n    sync.runtime_SemacquireWaitGroup(0x14197fd289c0?, 0xe0?)\n    \t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x14197ff32c88 sp=0x14197ff32c50 pc=0x1f3a66e\n    sync.(*WaitGroup).Wait(0x14197fc98608)\n    \t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x14197ff32cb0 sp=0x14197ff32c88 pc=0x1f4f445\n    golang.org/x/sync/errgroup.(*Group).Wait(0x14197fc98600)\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x14197ff32cd0 sp=0x14197ff32cb0 pc=0x362507e\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x14197f521dc0, {0x59f0b08, 0x14197f530fc0}, {0x14197fcca000, 0x15, 0x1c})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x14197ff32d90 sp=0x14197ff32cd0 pc=0x3927e05\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x14197f521dc0, {0x59f0b08, 0x14197f530fc0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:500 +0x4b08 fp=0x14197ff33968 sp=0x14197ff32d90 pc=0x39261c8\n    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedQueryOptions).RunQuery(0x14197f1401e0, {0x59f0b08, 0x14197f530fc0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_cmd.go:75 +0x565 fp=0x14197ff33b30 sp=0x14197ff33968 pc=0x3937005\n    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawQueryOptions).Run(0x0?, {0x59f0b08, 0x14197f530fc0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_options.go:178 +0x54 fp=0x14197ff33b58 sp=0x14197ff33b30 pc=0x3939d34\n    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryCommand.func1(0x14197f4ba008?, {0x14197f5f82a0?, 0x4?, 0x553b0e2?})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_cmd.go:45 +0x28 fp=0x14197ff33b80 sp=0x14197ff33b58 pc=0x3936a68\n    github.com/spf13/cobra.(*Command).execute(0x14197f4ba008, {0x14197f2c0030, 0xe, 0xf})\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x14197ff33d28 sp=0x14197ff33b80 pc=0x212e614\n    github.com/spf13/cobra.(*Command).ExecuteC(0x14197f625b08)\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x14197ff33e18 sp=0x14197ff33d28 pc=0x212efa5\n    github.com/spf13/cobra.(*Command).Execute(...)\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071\n    github.com/spf13/cobra.(*Command).ExecuteContext(...)\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064\n    main.main()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x14197ff33f48 sp=0x14197ff33e18 pc=0x47ddd6a\n    runtime.main()\n    \t/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x14197ff33fe0 sp=0x14197ff33f48 pc=0x1f01975\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197ff33fe8 sp=0x14197ff33fe0 pc=0x1f40801\n    \n    goroutine 2 gp=0x14197f132d20 m=nil [force gc (idle)]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f2fa8 sp=0x14197f1f2f88 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.forcegchelper()\n    \t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x14197f1f2fe0 sp=0x14197f1f2fa8 pc=0x1f01c93\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f2fe8 sp=0x14197f1f2fe0 pc=0x1f40801\n    created by runtime.init.7 in goroutine 1\n    \t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\n    \n    goroutine 18 gp=0x14197f2843c0 m=nil [GC sweep wait]:\n    runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1ee788 sp=0x14197f1ee768 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.bgsweep(0x14197f292000)\n    \t/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x14197f1ee7c8 sp=0x14197f1ee788 pc=0x1ee9f11\n    runtime.gcenable.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x14197f1ee7e0 sp=0x14197f1ee7c8 pc=0x1edadd7\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1ee7e8 sp=0x14197f1ee7e0 pc=0x1f40801\n    created by runtime.gcenable in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\n    \n    goroutine 19 gp=0x14197f2845a0 m=nil [GC scavenge wait]:\n    runtime.gopark(0x138d6d?, 0x3979fd?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1eef78 sp=0x14197f1eef58 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.(*scavengerState).park(0x82979c0)\n    \t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x14197f1eefa8 sp=0x14197f1eef78 pc=0x1ee77e9\n    runtime.bgscavenge(0x14197f292000)\n    \t/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x14197f1eefc8 sp=0x14197f1eefa8 pc=0x1ee7d99\n    runtime.gcenable.gowrap2()\n    \t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x14197f1eefe0 sp=0x14197f1eefc8 pc=0x1edad97\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1eefe8 sp=0x14197f1eefe0 pc=0x1f40801\n    created by runtime.gcenable in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\n    \n    goroutine 20 gp=0x14197f284780 m=nil [GOMAXPROCS updater (idle)]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1ef788 sp=0x14197f1ef768 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.updateMaxProcsGoroutine()\n    \t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x14197f1ef7e0 sp=0x14197f1ef788 pc=0x1f10267\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1ef7e8 sp=0x14197f1ef7e0 pc=0x1f40801\n    created by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\n    \t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\n    \n    goroutine 21 gp=0x14197f284d20 m=nil [finalizer wait]:\n    runtime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x14197f1f2601?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f2620 sp=0x14197f1f2600 pc=0x1f38d2e\n    runtime.runFinalizers()\n    \t/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x14197f1f27e0 sp=0x14197f1f2620 pc=0x1ed9cc7\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f27e8 sp=0x14197f1f27e0 pc=0x1f40801\n    created by runtime.createfing in goroutine 1\n    \t/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d\n    \n    goroutine 22 gp=0x14197f47c960 m=nil [cleanup wait]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1eff68 sp=0x14197f1eff48 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.(*cleanupQueue).dequeue(0x82980e0)\n    \t/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x14197f1effa0 sp=0x14197f1eff68 pc=0x1ed6074\n    runtime.runCleanups()\n    \t/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x14197f1effe0 sp=0x14197f1effa0 pc=0x1ed66e5\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1effe8 sp=0x14197f1effe0 pc=0x1f40801\n    created by runtime.(*cleanupQueue).createGs in goroutine 1\n    \t/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5\n    \n    goroutine 23 gp=0x14197f5a30e0 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125b862b4ec7?, 0x1?, 0x21?, 0xab?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f0740 sp=0x14197f1f0720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x14197f29d9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f07c8 sp=0x14197f1f0740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f07e0 sp=0x14197f1f07c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f07e8 sp=0x14197f1f07e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 3 gp=0x14197f133680 m=nil [runnable]:\n    runtime.gopark(0x125b8625d223?, 0x3?, 0x89?, 0x19?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f3740 sp=0x14197f1f3720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x14197f29d9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f37c8 sp=0x14197f1f3740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f37e0 sp=0x14197f1f37c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f37e8 sp=0x14197f1f37e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 34 gp=0x14197f604000 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125b862e8e02?, 0x3?, 0xe6?, 0x2d?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60a740 sp=0x14197f60a720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x14197f29d9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f60a7c8 sp=0x14197f60a740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f60a7e0 sp=0x14197f60a7c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60a7e8 sp=0x14197f60a7e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 24 gp=0x14197f5a32c0 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125b862118da?, 0x1?, 0x4d?, 0x9a?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f0f40 sp=0x14197f1f0f20 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x14197f29d9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f0fc8 sp=0x14197f1f0f40 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f0fe0 sp=0x14197f1f0fc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f0fe8 sp=0x14197f1f0fe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 4 gp=0x14197f133860 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125b7fe90140?, 0x3?, 0x20?, 0x3a?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f3f40 sp=0x14197f1f3f20 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x14197f29d9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f3fc8 sp=0x14197f1f3f40 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f3fe0 sp=0x14197f1f3fc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f3fe8 sp=0x14197f1f3fe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 35 gp=0x14197f6041e0 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125b862a34bd?, 0x3?, 0x4?, 0x15?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60af40 sp=0x14197f60af20 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x14197f29d9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f60afc8 sp=0x14197f60af40 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f60afe0 sp=0x14197f60afc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60afe8 sp=0x14197f60afe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 25 gp=0x14197f5a34a0 m=3 mp=0x14197f1a5808 [GC mark termination]:\n    runtime.systemstack_switch()\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x14197f1f16a0 sp=0x14197f1f1690 pc=0x1f3ebc8\n    runtime.forEachP(...)\n    \t/usr/lib/golang/src/runtime/proc.go:2112\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x14197f1f1740 sp=0x14197f1f16a0 pc=0x1edbc34\n    runtime.gcBgMarkWorker(0x14197f29d9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x14197f1f17c8 sp=0x14197f1f1740 pc=0x1edda88\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f17e0 sp=0x14197f1f17c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f17e8 sp=0x14197f1f17e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 5 gp=0x14197f133a40 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125b8624ea9b?, 0x1?, 0xfd?, 0x7a?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f4740 sp=0x14197f1f4720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x14197f29d9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f47c8 sp=0x14197f1f4740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f47e0 sp=0x14197f1f47c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f47e8 sp=0x14197f1f47e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 50 gp=0x14197f768f00 m=nil [select, locked to thread]:\n    runtime.gopark(0x14197f60c7a8?, 0x2?, 0x1?, 0x8f?, 0x14197f60c794?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60c628 sp=0x14197f60c608 pc=0x1f38d2e\n    runtime.selectgo(0x14197f60c7a8, 0x14197f60c790, 0x0?, 0x0, 0x0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f60c758 sp=0x14197f60c628 pc=0x1f152a5\n    runtime.ensureSigM.func1()\n    \t/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x14197f60c7e0 sp=0x14197f60c758 pc=0x1f328e8\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60c7e8 sp=0x14197f60c7e0 pc=0x1f40801\n    created by runtime.ensureSigM in goroutine 1\n    \t/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5\n    \n    goroutine 26 gp=0x14197f3b34a0 m=7 mp=0x14197f55b008 [syscall]:\n    runtime.notetsleepg(0x82bdea0, 0xffffffffffffffff)\n    \t/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x14197f6077a0 sp=0x14197f607778 pc=0x1ece509\n    os/signal.signal_recv()\n    \t/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x14197f6077c0 sp=0x14197f6077a0 pc=0x1f3b298\n    os/signal.loop()\n    \t/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x14197f6077e0 sp=0x14197f6077c0 pc=0x2030553\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f6077e8 sp=0x14197f6077e0 pc=0x1f40801\n    created by os/signal.Notify.func1.1 in goroutine 1\n    \t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\n    \n    goroutine 27 gp=0x14197f6894a0 m=nil [select]:\n    runtime.gopark(0x14197f607f98?, 0x2?, 0x68?, 0x81?, 0x14197f607f8c?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f607e28 sp=0x14197f607e08 pc=0x1f38d2e\n    runtime.selectgo(0x14197f607f98, 0x14197f607f88, 0x0?, 0x0, 0x0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f607f58 sp=0x14197f607e28 pc=0x1f152a5\n    os/signal.NotifyContext.func1()\n    \t/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x14197f607fe0 sp=0x14197f607f58 pc=0x202ff9c\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f607fe8 sp=0x14197f607fe0 pc=0x1f40801\n    created by os/signal.NotifyContext in goroutine 1\n    \t/usr/lib/golang/src/os/signal/signal.go:291 +0x158\n    \n    goroutine 224 gp=0x14197f689680 m=nil [semacquire]:\n    runtime.gopark(0x7f80904adba8?, 0x1eed7e0?, 0xc0?, 0x68?, 0x4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419800b5b58 sp=0x1419800b5b38 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x1419800b5bc0 sp=0x1419800b5b58 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x1419800b5c60 sp=0x1419800b5bc0 pc=0x1edbb12\n    runtime.gcAssistAlloc(0x14197f689680)\n    \t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x1419800b5cf0 sp=0x1419800b5c60 pc=0x1ee06b6\n    runtime.deductAssistCredit(0x7ffce51f9080?)\n    \t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x1419800b5d10 sp=0x1419800b5cf0 pc=0x1ed10a6\n    runtime.mallocgc(0x7fe00, 0x4af32a0, 0x1)\n    \t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x1419800b5d40 sp=0x1419800b5d10 pc=0x1f35e07\n    runtime.makeslice(0x14197fc9d0b0?, 0x141983a1fe00?, 0x20000?)\n    \t/usr/lib/golang/src/runtime/slice.go:117 +0x49 fp=0x1419800b5d68 sp=0x1419800b5d40 pc=0x1f3b489\n    encoding/json.(*Decoder).refill(0x14197f658c80)\n    \t/usr/lib/golang/src/encoding/json/stream.go:161 +0xee fp=0x1419800b5db8 sp=0x1419800b5d68 pc=0x200d48e\n    encoding/json.(*Decoder).readValue(0x14197f658c80)\n    \t/usr/lib/golang/src/encoding/json/stream.go:142 +0x85 fp=0x1419800b5e08 sp=0x1419800b5db8 pc=0x200d125\n    encoding/json.(*Decoder).Decode(0x14197f658c80, {0x4cfb340, 0x14197ffa44c0})\n    \t/usr/lib/golang/src/encoding/json/stream.go:65 +0x75 fp=0x1419800b5e38 sp=0x1419800b5e08 pc=0x200cdf5\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readPrimaryTable(0x14197f8c54f0, 0x14197fb50000)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:264 +0x16e fp=0x1419800b5ec8 sp=0x1419800b5e38 pc=0x38d822e\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x14197f8c54f0)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:174 +0x189 fp=0x1419800b5f30 sp=0x1419800b5ec8 pc=0x38d7b89\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x14197f8c54f0, 0x14198007be90)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x1419800b5fc0 sp=0x1419800b5f30 pc=0x38d788c\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x1419800b5fe0 sp=0x1419800b5fc0 pc=0x38d751b\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1419800b5fe8 sp=0x1419800b5fe0 pc=0x1f40801\n    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 183\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc\n    \n    goroutine 87 gp=0x14197fd3c000 m=nil [sync.WaitGroup.Wait]:\n    runtime.gopark(0x14197fcf9e00?, 0x1ed0734?, 0x20?, 0x94?, 0x7f80917bdf06?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf9da0 sp=0x14197fcf9d80 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x14197f69cc50, 0x0, 0x1, 0x0, 0x19)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fcf9e08 sp=0x14197fcf9da0 pc=0x1f162b2\n    sync.runtime_SemacquireWaitGroup(0x14197f86b2d8?, 0x0?)\n    \t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x14197fcf9e40 sp=0x14197fcf9e08 pc=0x1f3a66e\n    sync.(*WaitGroup).Wait(0x14197f69cc48)\n    \t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x14197fcf9e68 sp=0x14197fcf9e40 pc=0x1f4f445\n    golang.org/x/sync/errgroup.(*Group).Wait(0x14197f69cc40)\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x14197fcf9e88 sp=0x14197fcf9e68 pc=0x362507e\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x14197f521d80, {0x59f0b40, 0x14197f8c4b40}, {0x14197fcca000, 0x15, 0x6f63746669687372?}, 0x14197f1c02a0)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x14197fcf9f00 sp=0x14197fcf9e88 pc=0x392976a\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x14197fcf9f78 sp=0x14197fcf9f00 pc=0x392808e\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197fcf9fe0 sp=0x14197fcf9f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf9fe8 sp=0x14197fcf9fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 77 gp=0x14197fd3c1e0 m=nil [IO wait]:\n    runtime.gopark(0x14197fb3e4a0?, 0x1edb176?, 0x80?, 0xed?, 0xb?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f202808 sp=0x14197f2027e8 pc=0x1f38d2e\n    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n    \t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x14197f202840 sp=0x14197f202808 pc=0x1ef9e57\n    internal/poll.runtime_pollWait(0x7f8098109000, 0x72)\n    \t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x14197f202860 sp=0x14197f202840 pc=0x1f375c5\n    internal/poll.(*pollDesc).wait(0x14197f323a80?, 0x14197f34d300?, 0x0)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x14197f202888 sp=0x14197f202860 pc=0x1fbd167\n    internal/poll.(*pollDesc).waitRead(...)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\n    internal/poll.(*FD).Read(0x14197f323a80, {0x14197f34d300, 0x1300, 0x1300})\n    \t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x14197f202920 sp=0x14197f202888 pc=0x1fbe4ce\n    net.(*netFD).Read(0x14197f323a80, {0x14197f34d300?, 0x14197f34d30a?, 0x5?})\n    \t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x14197f202968 sp=0x14197f202920 pc=0x20798c5\n    net.(*conn).Read(0x14197f1300d8, {0x14197f34d300?, 0x7f809057a868?, 0x7f80df53a6d8?})\n    \t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x14197f2029b0 sp=0x14197f202968 pc=0x208b7a5\n    crypto/tls.(*atLeastReader).Read(0x14197f821a10, {0x14197f34d300?, 0x14197f202a98?, 0x1efcc28?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x14197f2029f8 sp=0x14197f2029b0 pc=0x2222ddb\n    bytes.(*Buffer).ReadFrom(0x14197f8a57a8, {0x59a3680, 0x14197f821a10})\n    \t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x14197f202a50 sp=0x14197f2029f8 pc=0x1fe9258\n    crypto/tls.(*Conn).readFromUntil(0x14197f8a5508, {0x59a32c0, 0x14197f1300d8}, 0x222764c?)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x14197f202a88 sp=0x14197f202a50 pc=0x2222fbe\n    crypto/tls.(*Conn).readRecordOrCCS(0x14197f8a5508, 0x0)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x14197f202c68 sp=0x14197f202a88 pc=0x222009b\n    crypto/tls.(*Conn).readRecord(...)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:588\n    crypto/tls.(*Conn).Read(0x14197f8a5508, {0x14197fa26000, 0x1000, 0x4e55540?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x14197f202cd0 sp=0x14197f202c68 pc=0x22267c5\n    net/http.(*persistConn).Read(0x14197f8032c0, {0x14197fa26000?, 0x59a0ea0?, 0x81260e0?})\n    \t/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x14197f202d30 sp=0x14197f202cd0 pc=0x2319d07\n    bufio.(*Reader).fill(0x14197f29b620)\n    \t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f202d68 sp=0x14197f202d30 pc=0x20a6c63\n    bufio.(*Reader).Peek(0x14197f29b620, 0x1)\n    \t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x14197f202d80 sp=0x14197f202d68 pc=0x20a6df3\n    net/http.(*persistConn).readLoop(0x14197f8032c0)\n    \t/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x14197f202fc8 sp=0x14197f202d80 pc=0x231a812\n    net/http.(*Transport).dialConn.gowrap2()\n    \t/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x14197f202fe0 sp=0x14197f202fc8 pc=0x2319197\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f202fe8 sp=0x14197f202fe0 pc=0x1f40801\n    created by net/http.(*Transport).dialConn in goroutine 9\n    \t/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac\n    \n    goroutine 7 gp=0x14197fe025a0 m=nil [sync.Mutex.Lock]:\n    runtime.gopark(0x14197f9d3c68?, 0x14197f9d3c70?, 0x94?, 0x69?, 0x14197f9d3c78?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f9d3c00 sp=0x14197f9d3be0 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x14197fccaad4, 0x0, 0x3, 0x2, 0x16)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197f9d3c68 sp=0x14197f9d3c00 pc=0x1f162b2\n    internal/sync.runtime_SemacquireMutex(0x1000?, 0xd0?, 0x1ed11a5?)\n    \t/usr/lib/golang/src/runtime/sema.go:95 +0x25 fp=0x14197f9d3ca0 sp=0x14197f9d3c68 pc=0x1f3a545\n    internal/sync.(*Mutex).lockSlow(0x14197fccaad0)\n    \t/usr/lib/golang/src/internal/sync/mutex.go:149 +0x15d fp=0x14197f9d3cf0 sp=0x14197f9d3ca0 pc=0x1f4d31d\n    internal/sync.(*Mutex).Lock(...)\n    \t/usr/lib/golang/src/internal/sync/mutex.go:70\n    sync.(*Mutex).Lock(...)\n    \t/usr/lib/golang/src/sync/mutex.go:46\n    net/http.(*http2clientConnReadLoop).streamByID(0x22babb5?, 0x1b, 0x1)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:10083 +0x5b fp=0x14197f9d3d48 sp=0x14197f9d3cf0 pc=0x22e10db\n    net/http.(*http2clientConnReadLoop).processData(0x14197f9d3fa8, 0x14197fc9d200)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9939 +0x45 fp=0x14197f9d3e18 sp=0x14197f9d3d48 pc=0x22e05e5\n    net/http.(*http2clientConnReadLoop).run(0x14197f9d3fa8)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9584 +0x4b6 fp=0x14197f9d3f60 sp=0x14197f9d3e18 pc=0x22de536\n    net/http.(*http2ClientConn).readLoop(0x14197fccaa80)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x14197f9d3fc8 sp=0x14197f9d3f60 pc=0x22dd532\n    net/http.(*http2Transport).newClientConn.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x14197f9d3fe0 sp=0x14197f9d3fc8 pc=0x22d7057\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f9d3fe8 sp=0x14197f9d3fe0 pc=0x1f40801\n    created by net/http.(*http2Transport).newClientConn in goroutine 6\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5\n    \n    goroutine 78 gp=0x14197fe02960 m=nil [select]:\n    runtime.gopark(0x14197f9f7f48?, 0x2?, 0x0?, 0xb6?, 0x14197f9f7ef4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf2d88 sp=0x14197fcf2d68 pc=0x1f38d2e\n    runtime.selectgo(0x14197fcf2f48, 0x14197f9f7ef0, 0x14197f1cd0c0?, 0x0, 0x14197f89c2a0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197fcf2eb8 sp=0x14197fcf2d88 pc=0x1f152a5\n    net/http.(*persistConn).writeLoop(0x14197f8032c0)\n    \t/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x14197fcf2fc8 sp=0x14197fcf2eb8 pc=0x231c426\n    net/http.(*Transport).dialConn.gowrap3()\n    \t/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x14197fcf2fe0 sp=0x14197fcf2fc8 pc=0x2319157\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf2fe8 sp=0x14197fcf2fe0 pc=0x1f40801\n    created by net/http.(*Transport).dialConn in goroutine 9\n    \t/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe\n    \n    goroutine 163 gp=0x14197f7694a0 m=nil [select]:\n    runtime.gopark(0x14197fdd3f08?, 0x6?, 0xa0?, 0x94?, 0x14197fdd3da4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f205c08 sp=0x14197f205be8 pc=0x1f38d2e\n    runtime.selectgo(0x14197f205f08, 0x14197fdd3d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f205d38 sp=0x14197f205c08 pc=0x1f152a5\n    net/http.(*http2clientStream).writeRequest(0x14197f37f380, 0x14197f49e140, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x14197f205f90 sp=0x14197f205d38 pc=0x22da6a8\n    net/http.(*http2clientStream).doRequest(0x14197f37f380, 0x14197f546100?, 0x14197f609798?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x14197f205fb8 sp=0x14197f205f90 pc=0x22d99d8\n    net/http.(*http2ClientConn).roundTrip.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197f205fe0 sp=0x14197f205fb8 pc=0x22d999f\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f205fe8 sp=0x14197f205fe0 pc=0x1f40801\n    created by net/http.(*http2ClientConn).roundTrip in goroutine 184\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n    \n    goroutine 88 gp=0x14197f769a40 m=nil [semacquire]:\n    runtime.gopark(0x141980119958?, 0x1ed3e05?, 0xe8?, 0xd2?, 0x7f80904ad2e8?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x141980119908 sp=0x1419801198e8 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x141980119970 sp=0x141980119908 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x141980119a10 sp=0x141980119970 pc=0x1edbb12\n    runtime.gcAssistAlloc(0x14197f769a40)\n    \t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x141980119aa0 sp=0x141980119a10 pc=0x1ee06b6\n    runtime.deductAssistCredit(0x141983b00000?)\n    \t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x141980119ac0 sp=0x141980119aa0 pc=0x1ed10a6\n    runtime.mallocgc(0x7edd, 0x0, 0x0)\n    \t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x141980119af0 sp=0x141980119ac0 pc=0x1f35e07\n    runtime.slicebytetostring(0x4e537a0?, 0x141983b00000, 0x7edd)\n    \t/usr/lib/golang/src/runtime/string.go:170 +0x77 fp=0x141980119b20 sp=0x141980119af0 pc=0x1f1e397\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x14197f8c4b40}, 0x14197f1c03f0, 0x14197f531740)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:322 +0x91b fp=0x141980119f48 sp=0x141980119b20 pc=0x3920bfb\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x141980119f78 sp=0x141980119f48 pc=0x392800c\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x141980119fe0 sp=0x141980119f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x141980119fe8 sp=0x141980119fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 110 gp=0x14197fd3cb40 m=nil [sync.Mutex.Lock]:\n    runtime.gopark(0x22b8a73?, 0x14197f7e1000?, 0xb0?, 0x4c?, 0x8160a40?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1c7510 sp=0x14197f1c74f0 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x14197fccaad4, 0x0, 0x3, 0x2, 0x16)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197f1c7578 sp=0x14197f1c7510 pc=0x1f162b2\n    internal/sync.runtime_SemacquireMutex(0x14197f1c75d0?, 0x66?, 0x61f?)\n    \t/usr/lib/golang/src/runtime/sema.go:95 +0x25 fp=0x14197f1c75b0 sp=0x14197f1c7578 pc=0x1f3a545\n    internal/sync.(*Mutex).lockSlow(0x14197fccaad0)\n    \t/usr/lib/golang/src/internal/sync/mutex.go:149 +0x15d fp=0x14197f1c7600 sp=0x14197f1c75b0 pc=0x1f4d31d\n    internal/sync.(*Mutex).Lock(...)\n    \t/usr/lib/golang/src/internal/sync/mutex.go:70\n    sync.(*Mutex).Lock(...)\n    \t/usr/lib/golang/src/sync/mutex.go:46\n    net/http.http2transportResponseBody.Read({0x1f3ec12?}, {0x14198313d000?, 0x1eed797?, 0x14197f1c76b0?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9875 +0x209 fp=0x14197f1c7688 sp=0x14197f1c7600 pc=0x22dff89\n    bufio.(*Reader).fill(0x14197f3a00c0)\n    \t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f1c76c0 sp=0x14197f1c7688 pc=0x20a6c63\n    bufio.(*Reader).ReadByte(0x14197f3a00c0)\n    \t/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x14197f1c76d8 sp=0x14197f1c76c0 pc=0x20a73c7\n    compress/flate.(*decompressor).moreBits(0x141980059908)\n    \t/usr/lib/golang/src/compress/flate/inflate.go:697 +0x24 fp=0x14197f1c7710 sp=0x14197f1c76d8 pc=0x2288824\n    compress/flate.(*decompressor).nextBlock(0x141980059908)\n    \t/usr/lib/golang/src/compress/flate/inflate.go:304 +0x28 fp=0x14197f1c7728 sp=0x14197f1c7710 pc=0x2286ba8\n    compress/flate.(*decompressor).Read(0x141980059908, {0x14198314e000, 0x1000, 0x7f8090ef0f00?})\n    \t/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x14197f1c7758 sp=0x14197f1c7728 pc=0x2286dbb\n    compress/gzip.(*Reader).Read(0x14197f672dc8, {0x14198314e000, 0x1000, 0x1000})\n    \t/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x14197f1c77e8 sp=0x14197f1c7758 pc=0x228c838\n    github.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x410d0fa76c5d7d40?, {0x14198314e000?, 0x7f8090ef0d20?, 0x52ffb40?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x14197f1c7818 sp=0x14197f1c77e8 pc=0x38b33e2\n    bufio.(*Reader).fill(0x14197f3a0120)\n    \t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f1c7850 sp=0x14197f1c7818 pc=0x20a6c63\n    bufio.(*Reader).Peek(0x14197f3a0120, 0x1)\n    \t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x14197f1c7868 sp=0x14197f1c7850 pc=0x20a6df3\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.validateJsonResponse(0x14197f3a0120)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:38 +0x25 fp=0x14197f1c78b8 sp=0x14197f1c7868 pc=0x38d6be5\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.newFrameReader({0x59db110, 0x14197fe1e040}, {0x59f0b40, 0x14197f748140})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:26 +0x188 fp=0x14197f1c7960 sp=0x14197f1c78b8 pc=0x38d6a08\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset({0x59f0bb0?, 0x14197f4797a0?}, {0x59db110, 0x14197fe1e040}, 0x1, 0x3e8, 0x1)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:66 +0x19e fp=0x14197f1c79e0 sp=0x14197f1c7960 pc=0x38d737e\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x14197f521d40, {0x59f0bb0, 0x14197f4797a0}, {0x14197f49a330, 0xb}, 0x14197fe1e700, {0x0?, 0x2?, 0x0?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:167 +0x2b7 fp=0x14197f1c7a78 sp=0x14197f1c79e0 pc=0x38e4f17\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197f7a2600}, 0x14197f1c02a0)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x14197f1c7d60 sp=0x14197f1c7a78 pc=0x391b5c5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x14197f521d80, {0x59f0b40, 0x14197fdd8140}, {0x5a00340, 0x14197fc1e5d0}, 0x14197f1c02a0)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:98 +0x366 fp=0x14197f1c7e88 sp=0x14197f1c7d60 pc=0x3929ec6\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14197f1c7f78 sp=0x14197f1c7e88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197f1c7fe0 sp=0x14197f1c7f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1c7fe8 sp=0x14197f1c7fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 89 gp=0x14197fd82960 m=nil [select]:\n    runtime.gopark(0x1419800b7d00?, 0x2?, 0xd7?, 0xfd?, 0x1419800b7c74?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419800b7ac8 sp=0x1419800b7aa8 pc=0x1f38d2e\n    runtime.selectgo(0x1419800b7d00, 0x1419800b7c70, 0x1419801521c0?, 0x0, 0x14197f5daba0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x1419800b7bf8 sp=0x1419800b7ac8 pc=0x1f152a5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x14197f521d80?, {0x59f0b40, 0x14197f8c4b40}, 0x14197f1c02a0, 0x14197f1c03f0)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x1419800b7f10 sp=0x1419800b7bf8 pc=0x392836e\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x1419800b7f78 sp=0x1419800b7f10 pc=0x3927f3d\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x1419800b7fe0 sp=0x1419800b7f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1419800b7fe8 sp=0x1419800b7fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 175 gp=0x14197fd3d2c0 m=nil [sync.Cond.Wait]:\n    runtime.gopark(0x22c2d5b?, 0x14197fe1c080?, 0x0?, 0xe0?, 0x14197f777000?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14198012b978 sp=0x14198012b958 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    sync.runtime_notifyListWait(0x14197f37f3c8, 0x3c)\n    \t/usr/lib/golang/src/runtime/sema.go:617 +0x1b3 fp=0x14198012b9c8 sp=0x14198012b978 pc=0x1f3a8f3\n    sync.(*Cond).Wait(0x1000?)\n    \t/usr/lib/golang/src/sync/cond.go:71 +0x73 fp=0x14198012ba00 sp=0x14198012b9c8 pc=0x1f4d673\n    net/http.(*http2pipe).Read(0x14197f37f3b0, {0x14197ff7e000, 0x1000, 0x1000})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:3928 +0xdb fp=0x14198012ba68 sp=0x14198012ba00 pc=0x22c2c1b\n    net/http.http2transportResponseBody.Read({0x4a54?}, {0x14197ff7e000?, 0x14198012bb80?, 0x2287949?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9852 +0x59 fp=0x14198012baf0 sp=0x14198012ba68 pc=0x22dfdd9\n    bufio.(*Reader).fill(0x14197fcaa000)\n    \t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14198012bb28 sp=0x14198012baf0 pc=0x20a6c63\n    bufio.(*Reader).ReadByte(0x14197fcaa000)\n    \t/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x14198012bb40 sp=0x14198012bb28 pc=0x20a73c7\n    compress/flate.(*decompressor).moreBits(0x141980056008)\n    \t/usr/lib/golang/src/compress/flate/inflate.go:697 +0x24 fp=0x14198012bb78 sp=0x14198012bb40 pc=0x2288824\n    compress/flate.(*decompressor).nextBlock(0x141980056008)\n    \t/usr/lib/golang/src/compress/flate/inflate.go:304 +0x28 fp=0x14198012bb90 sp=0x14198012bb78 pc=0x2286ba8\n    compress/flate.(*decompressor).Read(0x141980056008, {0x141980034b13, 0x4ed, 0xb13?})\n    \t/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x14198012bbc0 sp=0x14198012bb90 pc=0x2286dbb\n    compress/gzip.(*Reader).Read(0x141980000b08, {0x141980034b13, 0x4ed, 0x4ed})\n    \t/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x14198012bc50 sp=0x14198012bbc0 pc=0x228c838\n    github.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x14198012bcf0?, {0x141980034b13?, 0x1000?, 0x0?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x14198012bc80 sp=0x14198012bc50 pc=0x38b33e2\n    bufio.(*Reader).fill(0x141980807010)\n    \t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14198012bcb8 sp=0x14198012bc80 pc=0x20a6c63\n    bufio.(*Reader).ReadSlice(0x141980807010, 0xa)\n    \t/usr/lib/golang/src/bufio/bufio.go:377 +0x29 fp=0x14198012bd00 sp=0x14198012bcb8 pc=0x20a7769\n    bufio.(*Reader).collectFragments(0x141980807010, 0xa)\n    \t/usr/lib/golang/src/bufio/bufio.go:452 +0x85 fp=0x14198012bde0 sp=0x14198012bd00 pc=0x20a7b85\n    bufio.(*Reader).ReadBytes(0x4bbc5a0?, 0xa0?)\n    \t/usr/lib/golang/src/bufio/bufio.go:479 +0x17 fp=0x14198012be60 sp=0x14198012bde0 pc=0x20a7e77\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.(*frameReader).advance(0x141980807000)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:64 +0x45 fp=0x14198012beb0 sp=0x14198012be60 pc=0x38d6d45\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readRoutine(0x141980807000, 0x14197f8c8370)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:85 +0x51 fp=0x14198012bfc0 sp=0x14198012beb0 pc=0x38d7591\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap2()\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x1b fp=0x14198012bfe0 sp=0x14198012bfc0 pc=0x38d74db\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14198012bfe8 sp=0x14198012bfe0 pc=0x1f40801\n    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 184\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x24a\n    \n    goroutine 183 gp=0x14197fd3d860 m=nil [semacquire]:\n    runtime.gopark(0x7f8090f36c08?, 0x1eed7e0?, 0x80?, 0x43?, 0x2?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fac5208 sp=0x14197fac51e8 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fac5270 sp=0x14197fac5208 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x14197fac5310 sp=0x14197fac5270 pc=0x1edbb12\n    runtime.gcAssistAlloc(0x14197fd3d860)\n    \t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x14197fac53a0 sp=0x14197fac5310 pc=0x1ee06b6\n    runtime.deductAssistCredit(0x14197fac53f0?)\n    \t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x14197fac53c0 sp=0x14197fac53a0 pc=0x1ed10a6\n    runtime.mallocgc(0x5500, 0x0, 0x0)\n    \t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x14197fac53f0 sp=0x14197fac53c0 pc=0x1f35e07\n    runtime.growslice(0x141983adc000, 0x14197fac54a8?, 0x1ed3be5?, 0x82bed80?, 0x1ed0417?)\n    \t/usr/lib/golang/src/runtime/slice.go:265 +0x573 fp=0x14197fac5460 sp=0x14197fac53f0 pc=0x1f3ba93\n    strings.(*Builder).Write(0x14197f569ea0, {0x1419839fd000, 0x1000, 0x1000000010765?})\n    \t/usr/lib/golang/src/strings/builder.go:89 +0x90 fp=0x14197fac54b0 sp=0x14197fac5460 pc=0x1ff0cb0\n    bufio.(*Writer).Flush(0x14197fac5698)\n    \t/usr/lib/golang/src/bufio/bufio.go:642 +0x52 fp=0x14197fac54f8 sp=0x14197fac54b0 pc=0x20a8832\n    bufio.(*Writer).WriteString(0x14197fac5698, {0x553a4f6?, 0x553a9e6?})\n    \t/usr/lib/golang/src/bufio/bufio.go:766 +0x165 fp=0x14197fac5568 sp=0x14197fac54f8 pc=0x20a8f65\n    encoding/csv.(*Writer).Write(0x14197fac5688, {0x14198384d480?, 0x4, 0x141983909500?})\n    \t/usr/lib/golang/src/encoding/csv/writer.go:92 +0x2b4 fp=0x14197fac55d0 sp=0x14197fac5568 pc=0x20ad3d4\n    github.com/Azure/azure-kusto-go/azkustodata/query.(*row).String(0x5e?)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/row_impl.go:101 +0x21f fp=0x14197fac56e8 sp=0x14197fac55d0 pc=0x38cf67f\n    fmt.(*pp).handleMethods(0x14197fc20000, 0x7fac58f0?)\n    \t/usr/lib/golang/src/fmt/print.go:674 +0x2f4 fp=0x14197fac58a8 sp=0x14197fac56e8 pc=0x1fdd1d4\n    fmt.(*pp).printArg(0x14197fc20000, {0x549a3a0, 0x14197fd92e80}, 0x76)\n    \t/usr/lib/golang/src/fmt/print.go:757 +0x42e fp=0x14197fac5938 sp=0x14197fac58a8 pc=0x1fdd9ee\n    fmt.(*pp).doPrintf(0x14197fc20000, {0x553a400, 0x2}, {0x14197fac5d08, 0x1, 0x1})\n    \t/usr/lib/golang/src/fmt/print.go:1075 +0x3ec fp=0x14197fac5a20 sp=0x14197fac5938 pc=0x1fe04ec\n    fmt.Sprintf({0x553a400, 0x2}, {0x14197fac5d08, 0x1, 0x1})\n    \t/usr/lib/golang/src/fmt/print.go:239 +0x53 fp=0x14197fac5a78 sp=0x14197fac5a20 pc=0x1fda6b3\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197fc4b200}, 0x14197f1c02a0)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:160 +0x8d8 fp=0x14197fac5d60 sp=0x14197fac5a78 pc=0x391bad8\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x14197fdd8140?}, {0x5a00300?, 0x14197fc4b200?}, 0x0?)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x14197fac5e88 sp=0x14197fac5d60 pc=0x3929c16\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14197fac5f78 sp=0x14197fac5e88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197fac5fe0 sp=0x14197fac5f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fac5fe8 sp=0x14197fac5fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 184 gp=0x14197fd3da40 m=nil [chan receive]:\n    runtime.gopark(0x14197f3c29a0?, 0x2034990?, 0x10?, 0xea?, 0x59f0ad0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14198011f9d8 sp=0x14198011f9b8 pc=0x1f38d2e\n    runtime.chanrecv(0x14197f3fa700, 0x14198011fcb8, 0x1)\n    \t/usr/lib/golang/src/runtime/chan.go:667 +0x4ae fp=0x14198011fa50 sp=0x14198011f9d8 pc=0x1ec7d2e\n    runtime.chanrecv2(0x5a09ed0?, 0x14197f530f90?)\n    \t/usr/lib/golang/src/runtime/chan.go:514 +0x12 fp=0x14198011fa78 sp=0x14198011fa50 pc=0x1ec7872\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197fc4b340}, 0x14197f1c02a0)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:141 +0x94d fp=0x14198011fd60 sp=0x14198011fa78 pc=0x391bb4d\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x74666968736e6570?, {0x59f0b40?, 0x14197fdd8140?}, {0x5a00300?, 0x14197fc4b340?}, 0x6d736f63222c2273?)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x14198011fe88 sp=0x14198011fd60 pc=0x3929c16\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14198011ff78 sp=0x14198011fe88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14198011ffe0 sp=0x14198011ff78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14198011ffe8 sp=0x14198011ffe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 135 gp=0x14197fb465a0 m=nil [semacquire]:\n    runtime.gopark(0x14197f454000?, 0x22dc6c0?, 0xa0?, 0x82?, 0x1419808322a0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fe99ce8 sp=0x14197fe99cc8 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fe99d50 sp=0x14197fe99ce8 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x14197fe99df0 sp=0x14197fe99d50 pc=0x1edbb12\n    runtime.gcAssistAlloc(0x14197fb465a0)\n    \t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x14197fe99e80 sp=0x14197fe99df0 pc=0x1ee06b6\n    runtime.deductAssistCredit(0x0?)\n    \t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x14197fe99ea0 sp=0x14197fe99e80 pc=0x1ed10a6\n    runtime.mallocgc(0x70, 0x0, 0x1)\n    \t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x14197fe99ed0 sp=0x14197fe99ea0 pc=0x1f35e07\n    runtime.makechan(0x0?, 0x0)\n    \t/usr/lib/golang/src/runtime/chan.go:99 +0x126 fp=0x14197fe99f10 sp=0x14197fe99ed0 pc=0x1ec6826\n    net/http.(*http2clientStream).cleanupWriteRequest(0x14197f2f1200, {0x0, 0x0})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8951 +0xd6 fp=0x14197fe99f90 sp=0x14197fe99f10 pc=0x22db1b6\n    net/http.(*http2clientStream).doRequest(0x14197f2f1200, 0x14197f7480a0?, 0x14197f4d0020?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8719 +0x28 fp=0x14197fe99fb8 sp=0x14197fe99f90 pc=0x22d99e8\n    net/http.(*http2ClientConn).roundTrip.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197fe99fe0 sp=0x14197fe99fb8 pc=0x22d999f\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fe99fe8 sp=0x14197fe99fe0 pc=0x1f40801\n    created by net/http.(*http2ClientConn).roundTrip in goroutine 110\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n    \n    goroutine 90 gp=0x14197fd82b40 m=nil [select]:\n    runtime.gopark(0x14197ff1df08?, 0x6?, 0x40?, 0x2b?, 0x14197ff1dda4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f203c08 sp=0x14197f203be8 pc=0x1f38d2e\n    runtime.selectgo(0x14197f203f08, 0x14197ff1dd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f203d38 sp=0x14197f203c08 pc=0x1f152a5\n    net/http.(*http2clientStream).writeRequest(0x14197f2f0f00, 0x14197fb50140, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x14197f203f90 sp=0x14197f203d38 pc=0x22da6a8\n    net/http.(*http2clientStream).doRequest(0x14197f2f0f00, 0x0?, 0x14197f1f1f98?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x14197f203fb8 sp=0x14197f203f90 pc=0x22d99d8\n    net/http.(*http2ClientConn).roundTrip.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197f203fe0 sp=0x14197f203fb8 pc=0x22d999f\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f203fe8 sp=0x14197f203fe0 pc=0x1f40801\n    created by net/http.(*http2ClientConn).roundTrip in goroutine 183\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n    \n    goroutine 225 gp=0x14197fd834a0 m=nil [sync.Cond.Wait]:\n    runtime.gopark(0x22b8a73?, 0x141980390000?, 0x0?, 0x60?, 0xe?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419801298f8 sp=0x1419801298d8 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    sync.runtime_notifyListWait(0x14197f2f0f48, 0x3c)\n    \t/usr/lib/golang/src/runtime/sema.go:617 +0x1b3 fp=0x141980129948 sp=0x1419801298f8 pc=0x1f3a8f3\n    sync.(*Cond).Wait(0x14197fbb1640?)\n    \t/usr/lib/golang/src/sync/cond.go:71 +0x73 fp=0x141980129980 sp=0x141980129948 pc=0x1f4d673\n    net/http.(*http2pipe).Read(0x14197f2f0f30, {0x14197fd98000, 0x1000, 0x1000})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:3928 +0xdb fp=0x1419801299e8 sp=0x141980129980 pc=0x22c2c1b\n    net/http.http2transportResponseBody.Read({0x141980129a60?}, {0x14197fd98000?, 0x20a6c63?, 0x1f3ec12?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9852 +0x59 fp=0x141980129a70 sp=0x1419801299e8 pc=0x22dfdd9\n    bufio.(*Reader).fill(0x14197fc90180)\n    \t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x141980129aa8 sp=0x141980129a70 pc=0x20a6c63\n    bufio.(*Reader).ReadByte(0x14197fc90180)\n    \t/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x141980129ac0 sp=0x141980129aa8 pc=0x20a73c7\n    compress/flate.(*decompressor).huffSym(0x141980058608, 0x141980058638)\n    \t/usr/lib/golang/src/compress/flate/inflate.go:720 +0x7e fp=0x141980129b10 sp=0x141980129ac0 pc=0x228893e\n    compress/flate.(*decompressor).huffmanBlock(0x82bed80?)\n    \t/usr/lib/golang/src/compress/flate/inflate.go:495 +0x45 fp=0x141980129b90 sp=0x141980129b10 pc=0x2287605\n    compress/flate.(*decompressor).Read(0x141980058608, {0x14197fd99cea, 0x316, 0xcea?})\n    \t/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x141980129bc0 sp=0x141980129b90 pc=0x2286dbb\n    compress/gzip.(*Reader).Read(0x14197f3d8848, {0x14197fd99cea, 0x316, 0x316})\n    \t/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x141980129c50 sp=0x141980129bc0 pc=0x228c838\n    github.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x141980129cf0?, {0x14197fd99cea?, 0x1000?, 0x0?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x141980129c80 sp=0x141980129c50 pc=0x38b33e2\n    bufio.(*Reader).fill(0x14197f546110)\n    \t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x141980129cb8 sp=0x141980129c80 pc=0x20a6c63\n    bufio.(*Reader).ReadSlice(0x14197f546110, 0xa)\n    \t/usr/lib/golang/src/bufio/bufio.go:377 +0x29 fp=0x141980129d00 sp=0x141980129cb8 pc=0x20a7769\n    bufio.(*Reader).collectFragments(0x14197f546110, 0xa)\n    \t/usr/lib/golang/src/bufio/bufio.go:452 +0x85 fp=0x141980129de0 sp=0x141980129d00 pc=0x20a7b85\n    bufio.(*Reader).ReadBytes(0x4bbc5a0?, 0xa0?)\n    \t/usr/lib/golang/src/bufio/bufio.go:479 +0x17 fp=0x141980129e60 sp=0x141980129de0 pc=0x20a7e77\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.(*frameReader).advance(0x14197f546100)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:64 +0x45 fp=0x141980129eb0 sp=0x141980129e60 pc=0x38d6d45\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readRoutine(0x14197f546100, 0x14197f8c54f0)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:85 +0x51 fp=0x141980129fc0 sp=0x141980129eb0 pc=0x38d7591\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap2()\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x1b fp=0x141980129fe0 sp=0x141980129fc0 pc=0x38d74db\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x141980129fe8 sp=0x141980129fe0 pc=0x1f40801\n    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 183\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x24a\n    \n    goroutine 174 gp=0x14197ff64000 m=nil [select]:\n    runtime.gopark(0x1419800b9df8?, 0x2?, 0x0?, 0x0?, 0x1419800b9db4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf3c50 sp=0x14197fcf3c30 pc=0x1f38d2e\n    runtime.selectgo(0x14197fcf3df8, 0x1419800b9db0, 0x59d87c8?, 0x0, 0x14197f2980e0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197fcf3d80 sp=0x14197fcf3c50 pc=0x1f152a5\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.nextFrame(0x14197f8c8370)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:198 +0x96 fp=0x14197fcf3e38 sp=0x14197fcf3d80 pc=0x38d7d16\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readPrimaryTable(0x14197f8c8370, 0x14197fb50000)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:258 +0xb1 fp=0x14197fcf3ec8 sp=0x14197fcf3e38 pc=0x38d8171\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x14197f8c8370)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:174 +0x189 fp=0x14197fcf3f30 sp=0x14197fcf3ec8 pc=0x38d7b89\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x14197f8c8370, 0x14197f59aee0)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x14197fcf3fc0 sp=0x14197fcf3f30 pc=0x38d788c\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x14197fcf3fe0 sp=0x14197fcf3fc0 pc=0x38d751b\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf3fe8 sp=0x14197fcf3fe0 pc=0x1f40801\n    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 184\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc\n    \n    rax    0x0\n    rbx    0x7f808bfff640\n    rcx    0x7f80df60594c\n    rdx    0x6\n    rdi    0x109fd\n    rsi    0x10a00\n    rbp    0x10a00\n    rsp    0x7f808bffe790\n    r8     0x7f808bffe860\n    r9     0x7f80df72b4e0\n    r10    0x8\n    r11    0x246\n    r12    0x6\n    r13    0x7f808bffeb60\n    r14    0x14197f1334a0\n    r15    0xffffffffffffffff\n    rip    0x7f80df60594c\n    rflags 0x246\n    cs     0x33\n    fs     0x0\n    gs     0x0\n    \n      query-infra-svc: command failed: exit status 2\n    output: runtime/cgo: pthread_create failed: Resource temporarily unavailable\n    SIGABRT: abort\n    PC=0x7f7ef9baa94c m=3 sigcode=18446744073709551610\n    \n    goroutine 0 gp=0x170ac5c1f0e0 m=3 mp=0x170ac5c91808 [idle]:\n    runtime: g 0 gp=0x170ac5c1f0e0: unknown pc 0x7f7ef9baa94c\n    stack: frame={sp:0x7f7eabffe790, fp:0x0} stack=[0x7f7eab7ff1e0,0x7f7eabffede0)\n                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n    00007f7eabffe690: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6a0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6b0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6c0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6d0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6e0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6f0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe700: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe710: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe720: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe730: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe740: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe750: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe760: 00000000 00000000  00007f7e f9bb7fa9  ............~...\n    00007f7eabffe770: 00000000 00000000  00000000 00000011  ................\n    00007f7eabffe780: 00000000 00000130  00007f7e f9baa93e  0.......\u003e...~...\n    00007f7eabffe790: 00000000 00000120  00000000 00000000   ...............\n                      ^ SP\n    00007f7eabffe7a0: 00000000 00000013  00000000 00000000  ................\n    00007f7eabffe7b0: 00000000 00000000  00007f7e f9c2dcff  ............~...\n    00007f7eabffe7c0: 00007f7e 7bfff910  00007f7e f9c2dcb1  ...{~.......~...\n    00007f7eabffe7d0: 00007f7e 7bfff640  00007f7e abffeaa0  @..{~.......~...\n    00007f7eabffe7e0: 00007f7e abffe90e  00007f7e abffe90f  ....~.......~...\n    00007f7eabffe7f0: 00007f7e 7bfff640  00007f7e f9ba87f5  @..{~.......~...\n    00007f7eabffe800: 00007f7e a4000020  00000000 00000120   ...~... .......\n    00007f7eabffe810: 00000000 003d0f00  176a34c4 bac87500  ..=......u...4j.\n    00007f7eabffe820: 00007f7e abfff640  00000000 00000006  @...~...........\n    00007f7eabffe830: 00007f7e a4000f30  00007f7e abffeb60  0...~...`...~...\n    00007f7eabffe840: 0000170a c5c1f0e0  00007f7e f9b5d646  ........F...~...\n    00007f7eabffe850: 00007f7e f9d1ae90  00007f7e f9b477f3  ....~....w..~...\n    00007f7eabffe860: 00000000 00000020  176a34c4 bac87500   ........u...4j.\n    00007f7eabffe870: 00007f7e 7b7ff000  00007f7e f9c1cac4  ...{~.......~...\n    00007f7eabffe880: 00000000 00000000  00000000 0000000d  ................\n    runtime: g 0 gp=0x170ac5c1f0e0: unknown pc 0x7f7ef9baa94c\n    stack: frame={sp:0x7f7eabffe790, fp:0x0} stack=[0x7f7eab7ff1e0,0x7f7eabffede0)\n                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n    00007f7eabffe690: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6a0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6b0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6c0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6d0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6e0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe6f0: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe700: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe710: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe720: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe730: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe740: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe750: 00000000 00000000  00000000 00000000  ................\n    00007f7eabffe760: 00000000 00000000  00007f7e f9bb7fa9  ............~...\n    00007f7eabffe770: 00000000 00000000  00000000 00000011  ................\n    00007f7eabffe780: 00000000 00000130  00007f7e f9baa93e  0.......\u003e...~...\n    00007f7eabffe790: 00000000 00000120  00000000 00000000   ...............\n                      ^ SP\n    00007f7eabffe7a0: 00000000 00000013  00000000 00000000  ................\n    00007f7eabffe7b0: 00000000 00000000  00007f7e f9c2dcff  ............~...\n    00007f7eabffe7c0: 00007f7e 7bfff910  00007f7e f9c2dcb1  ...{~.......~...\n    00007f7eabffe7d0: 00007f7e 7bfff640  00007f7e abffeaa0  @..{~.......~...\n    00007f7eabffe7e0: 00007f7e abffe90e  00007f7e abffe90f  ....~.......~...\n    00007f7eabffe7f0: 00007f7e 7bfff640  00007f7e f9ba87f5  @..{~.......~...\n    00007f7eabffe800: 00007f7e a4000020  00000000 00000120   ...~... .......\n    00007f7eabffe810: 00000000 003d0f00  176a34c4 bac87500  ..=......u...4j.\n    00007f7eabffe820: 00007f7e abfff640  00000000 00000006  @...~...........\n    00007f7eabffe830: 00007f7e a4000f30  00007f7e abffeb60  0...~...`...~...\n    00007f7eabffe840: 0000170a c5c1f0e0  00007f7e f9b5d646  ........F...~...\n    00007f7eabffe850: 00007f7e f9d1ae90  00007f7e f9b477f3  ....~....w..~...\n    00007f7eabffe860: 00000000 00000020  176a34c4 bac87500   ........u...4j.\n    00007f7eabffe870: 00007f7e 7b7ff000  00007f7e f9c1cac4  ...{~.......~...\n    00007f7eabffe880: 00000000 00000000  00000000 0000000d  ................\n    \n    goroutine 1 gp=0x170ac5c1e1e0 m=nil [sync.WaitGroup.Wait]:\n    runtime.gopark(0x170ac5cb27b8?, 0x2022a4d?, 0x70?, 0x8e?, 0x170ac5cb27e8?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cb27a0 sp=0x170ac5cb2780 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x170ac65f8210, 0x0, 0x1, 0x0, 0x19)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x170ac5cb2808 sp=0x170ac5cb27a0 pc=0x1f162b2\n    sync.runtime_SemacquireWaitGroup(0x170ac6709950?, 0xe0?)\n    \t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x170ac5cb2840 sp=0x170ac5cb2808 pc=0x1f3a66e\n    sync.(*WaitGroup).Wait(0x170ac65f8208)\n    \t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x170ac5cb2868 sp=0x170ac5cb2840 pc=0x1f4f445\n    golang.org/x/sync/errgroup.(*Group).Wait(0x170ac65f8200)\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x170ac5cb2888 sp=0x170ac5cb2868 pc=0x362507e\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0}, {0x170ac626a1e0, 0x6, 0x6})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x170ac5cb2948 sp=0x170ac5cb2888 pc=0x3927e05\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).gatherInfraLogs(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:595 +0xadd fp=0x170ac5cb2d30 sp=0x170ac5cb2948 pc=0x392799d\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:333 +0x533 fp=0x170ac5cb3908 sp=0x170ac5cb2d30 pc=0x3921bf3\n    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedInfraQueryOptions).RunInfra(0x170ac5c2c108, {0x59f0b08, 0x170ac64108d0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:83 +0x71f fp=0x170ac5cb3b30 sp=0x170ac5cb3908 pc=0x393887f\n    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawInfraQueryOptions).Run(0x0?, {0x59f0b08, 0x170ac64108d0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_options.go:109 +0x54 fp=0x170ac5cb3b58 sp=0x170ac5cb3b30 pc=0x3939134\n    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryInfraCommand.func1(0x170ac66f5508?, {0x170ac5d40960?, 0x4?, 0x553b0e2?})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:45 +0x28 fp=0x170ac5cb3b80 sp=0x170ac5cb3b58 pc=0x3938128\n    github.com/spf13/cobra.(*Command).execute(0x170ac66f5508, {0x170ac5e36100, 0xa, 0xa})\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x170ac5cb3d28 sp=0x170ac5cb3b80 pc=0x212e614\n    github.com/spf13/cobra.(*Command).ExecuteC(0x170ac6228608)\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x170ac5cb3e18 sp=0x170ac5cb3d28 pc=0x212efa5\n    github.com/spf13/cobra.(*Command).Execute(...)\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071\n    github.com/spf13/cobra.(*Command).ExecuteContext(...)\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064\n    main.main()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x170ac5cb3f48 sp=0x170ac5cb3e18 pc=0x47ddd6a\n    runtime.main()\n    \t/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x170ac5cb3fe0 sp=0x170ac5cb3f48 pc=0x1f01975\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cb3fe8 sp=0x170ac5cb3fe0 pc=0x1f40801\n    \n    goroutine 2 gp=0x170ac5c1ed20 m=nil [force gc (idle)]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdefa8 sp=0x170ac5cdef88 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.forcegchelper()\n    \t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x170ac5cdefe0 sp=0x170ac5cdefa8 pc=0x1f01c93\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdefe8 sp=0x170ac5cdefe0 pc=0x1f40801\n    created by runtime.init.7 in goroutine 1\n    \t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\n    \n    goroutine 18 gp=0x170ac5d843c0 m=nil [GC sweep wait]:\n    runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cda788 sp=0x170ac5cda768 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.bgsweep(0x170ac5d92000)\n    \t/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x170ac5cda7c8 sp=0x170ac5cda788 pc=0x1ee9f11\n    runtime.gcenable.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x170ac5cda7e0 sp=0x170ac5cda7c8 pc=0x1edadd7\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cda7e8 sp=0x170ac5cda7e0 pc=0x1f40801\n    created by runtime.gcenable in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\n    \n    goroutine 19 gp=0x170ac5d845a0 m=nil [GC scavenge wait]:\n    runtime.gopark(0x10000?, 0x5981cd8?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdaf78 sp=0x170ac5cdaf58 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.(*scavengerState).park(0x82979c0)\n    \t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x170ac5cdafa8 sp=0x170ac5cdaf78 pc=0x1ee77e9\n    runtime.bgscavenge(0x170ac5d92000)\n    \t/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x170ac5cdafc8 sp=0x170ac5cdafa8 pc=0x1ee7d99\n    runtime.gcenable.gowrap2()\n    \t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x170ac5cdafe0 sp=0x170ac5cdafc8 pc=0x1edad97\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdafe8 sp=0x170ac5cdafe0 pc=0x1f40801\n    created by runtime.gcenable in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\n    \n    goroutine 34 gp=0x170ac5e00000 m=nil [GOMAXPROCS updater (idle)]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e06788 sp=0x170ac5e06768 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.updateMaxProcsGoroutine()\n    \t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x170ac5e067e0 sp=0x170ac5e06788 pc=0x1f10267\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e067e8 sp=0x170ac5e067e0 pc=0x1f40801\n    created by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\n    \t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\n    \n    goroutine 35 gp=0x170ac5e005a0 m=nil [finalizer wait]:\n    runtime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x170ac5cde601?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cde620 sp=0x170ac5cde600 pc=0x1f38d2e\n    runtime.runFinalizers()\n    \t/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x170ac5cde7e0 sp=0x170ac5cde620 pc=0x1ed9cc7\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cde7e8 sp=0x170ac5cde7e0 pc=0x1f40801\n    created by runtime.createfing in goroutine 1\n    \t/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d\n    \n    goroutine 36 gp=0x170ac5ff81e0 m=nil [cleanup wait]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e06f68 sp=0x170ac5e06f48 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.(*cleanupQueue).dequeue(0x82980e0)\n    \t/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x170ac5e06fa0 sp=0x170ac5e06f68 pc=0x1ed6074\n    runtime.runCleanups()\n    \t/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x170ac5e06fe0 sp=0x170ac5e06fa0 pc=0x1ed66e5\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e06fe8 sp=0x170ac5e06fe0 pc=0x1f40801\n    created by runtime.(*cleanupQueue).createGs in goroutine 1\n    \t/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5\n    \n    goroutine 37 gp=0x170ac611bc20 m=nil [GC worker (idle)]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e07740 sp=0x170ac5e07720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x170ac5e0b9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5e077c8 sp=0x170ac5e07740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5e077e0 sp=0x170ac5e077c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e077e8 sp=0x170ac5e077e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 3 gp=0x170ac5c1f680 m=nil [runnable]:\n    runtime.gopark(0x125c26559f5b?, 0x3?, 0x94?, 0x85?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdf740 sp=0x170ac5cdf720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x170ac5e0b9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdf7c8 sp=0x170ac5cdf740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdf7e0 sp=0x170ac5cdf7c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdf7e8 sp=0x170ac5cdf7e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 20 gp=0x170ac5d84780 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125c265c3db1?, 0x3?, 0x92?, 0xcd?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdb740 sp=0x170ac5cdb720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x170ac5e0b9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdb7c8 sp=0x170ac5cdb740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdb7e0 sp=0x170ac5cdb7c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdb7e8 sp=0x170ac5cdb7e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 38 gp=0x170ac6134000 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125c26588b9e?, 0x1?, 0xf7?, 0xb4?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e07f40 sp=0x170ac5e07f20 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x170ac5e0b9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5e07fc8 sp=0x170ac5e07f40 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5e07fe0 sp=0x170ac5e07fc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e07fe8 sp=0x170ac5e07fe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 4 gp=0x170ac5c1f860 m=10 mp=0x170ac6280008 [GC mark termination]:\n    runtime.systemstack_switch()\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x170ac5cdfea0 sp=0x170ac5cdfe90 pc=0x1f3ebc8\n    runtime.forEachP(...)\n    \t/usr/lib/golang/src/runtime/proc.go:2112\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x170ac5cdff40 sp=0x170ac5cdfea0 pc=0x1edbc34\n    runtime.gcBgMarkWorker(0x170ac5e0b9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x170ac5cdffc8 sp=0x170ac5cdff40 pc=0x1edda88\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdffe0 sp=0x170ac5cdffc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdffe8 sp=0x170ac5cdffe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 21 gp=0x170ac5d84960 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125c265818cf?, 0x3?, 0xa5?, 0x9c?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdbf40 sp=0x170ac5cdbf20 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x170ac5e0b9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdbfc8 sp=0x170ac5cdbf40 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdbfe0 sp=0x170ac5cdbfc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdbfe8 sp=0x170ac5cdbfe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 22 gp=0x170ac5d84b40 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125c265846e7?, 0x3?, 0x77?, 0xb6?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdc740 sp=0x170ac5cdc720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x170ac5e0b9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdc7c8 sp=0x170ac5cdc740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdc7e0 sp=0x170ac5cdc7c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdc7e8 sp=0x170ac5cdc7e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 23 gp=0x170ac5d84d20 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125c26581951?, 0x3?, 0x91?, 0x58?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdcf40 sp=0x170ac5cdcf20 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x170ac5e0b9d0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdcfc8 sp=0x170ac5cdcf40 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdcfe0 sp=0x170ac5cdcfc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdcfe8 sp=0x170ac5cdcfe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 39 gp=0x170ac648a3c0 m=nil [select, locked to thread]:\n    runtime.gopark(0x170ac5e097a8?, 0x2?, 0x1?, 0xa3?, 0x170ac5e09794?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e09628 sp=0x170ac5e09608 pc=0x1f38d2e\n    runtime.selectgo(0x170ac5e097a8, 0x170ac5e09790, 0x0?, 0x0, 0x0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5e09758 sp=0x170ac5e09628 pc=0x1f152a5\n    runtime.ensureSigM.func1()\n    \t/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x170ac5e097e0 sp=0x170ac5e09758 pc=0x1f328e8\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e097e8 sp=0x170ac5e097e0 pc=0x1f40801\n    created by runtime.ensureSigM in goroutine 1\n    \t/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5\n    \n    goroutine 24 gp=0x170ac62f4960 m=8 mp=0x170ac6180008 [syscall]:\n    runtime.notetsleepg(0x82bdea0, 0xffffffffffffffff)\n    \t/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x170ac5e037a0 sp=0x170ac5e03778 pc=0x1ece509\n    os/signal.signal_recv()\n    \t/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x170ac5e037c0 sp=0x170ac5e037a0 pc=0x1f3b298\n    os/signal.loop()\n    \t/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x170ac5e037e0 sp=0x170ac5e037c0 pc=0x2030553\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e037e8 sp=0x170ac5e037e0 pc=0x1f40801\n    created by os/signal.Notify.func1.1 in goroutine 1\n    \t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\n    \n    goroutine 50 gp=0x170ac6135a40 m=nil [select]:\n    runtime.gopark(0x170ac66e3798?, 0x2?, 0x80?, 0x6f?, 0x170ac66e378c?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac66e3628 sp=0x170ac66e3608 pc=0x1f38d2e\n    runtime.selectgo(0x170ac66e3798, 0x170ac66e3788, 0x0?, 0x0, 0x0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac66e3758 sp=0x170ac66e3628 pc=0x1f152a5\n    os/signal.NotifyContext.func1()\n    \t/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x170ac66e37e0 sp=0x170ac66e3758 pc=0x202ff9c\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac66e37e8 sp=0x170ac66e37e0 pc=0x1f40801\n    created by os/signal.NotifyContext in goroutine 1\n    \t/usr/lib/golang/src/os/signal/signal.go:291 +0x158\n    \n    goroutine 102 gp=0x170ac61843c0 m=nil [select]:\n    runtime.gopark(0x170ac5cefc70?, 0x2?, 0xc8?, 0xf9?, 0x170ac5cefb6c?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cef9f0 sp=0x170ac5cef9d0 pc=0x1f38d2e\n    runtime.selectgo(0x170ac5cefc70, 0x170ac5cefb68, 0x554857c?, 0x0, 0x226?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5cefb20 sp=0x170ac5cef9f0 pc=0x1f152a5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x170ac5c49130}, 0x170ac65fa070, 0x170ac6410cf0)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:283 +0x21b fp=0x170ac5ceff48 sp=0x170ac5cefb20 pc=0x39204fb\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x170ac5ceff78 sp=0x170ac5ceff48 pc=0x392800c\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5ceffe0 sp=0x170ac5ceff78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5ceffe8 sp=0x170ac5ceffe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 104 gp=0x170ac6184d20 m=nil [select]:\n    runtime.gopark(0x170ac6992ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69929d8?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6992848 sp=0x170ac6992828 pc=0x1f38d2e\n    runtime.selectgo(0x170ac6992ac8, 0x170ac69929d0, 0x55562e1?, 0x0, 0x1f6d6cf?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6992978 sp=0x170ac6992848 pc=0x1f152a5\n    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac619fb80, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac6992b40 sp=0x170ac6992978 pc=0x22d92e5\n    net/http.(*http2ClientConn).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8603\n    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac619fb80, {0x4c?, 0xfa?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac6992c80 sp=0x170ac6992b40 pc=0x22d50ad\n    net/http.(*http2Transport).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7852\n    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac619fb80?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac6992cc0 sp=0x170ac6992c80 pc=0x22e3a9c\n    net/http.(*Transport).roundTrip(0x8175400, 0x170ac619fa40)\n    \t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac6992ea0 sp=0x170ac6992cc0 pc=0x231037f\n    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n    \t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac6992ec0 sp=0x170ac6992ea0 pc=0x2321178\n    net/http.send(0x170ac619fa40, {0x59a1ce0, 0x8175400}, {0x170ac6993138?, 0x1f3bac6?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69930b0 sp=0x170ac6992ec0 pc=0x22ad9ab\n    net/http.(*Client).send(0x170ac6410c60, 0x170ac619fa40, {0x170ac69931e8?, 0x1fdd787?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac6993148 sp=0x170ac69930b0 pc=0x22ad1d8\n    net/http.(*Client).do(0x170ac6410c60, 0x170ac619fa40)\n    \t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac6993338 sp=0x170ac6993148 pc=0x22af817\n    net/http.(*Client).Do(...)\n    \t/usr/lib/golang/src/net/http/client.go:592\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5c49450}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e34a0}, 0x170ac65f7ce0, {0x170ac65d9560, 0x112})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac6993630 sp=0x170ac6993338 pc=0x38de6bd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5c49450}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x170ac65f7c80, 0x0, {0x0, ...}, ...})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac6993888 sp=0x170ac6993630 pc=0x38ddecd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5c49450?}, 0xa8?, {0x170ac5db1a50?, 0x170ac6764a48?}, 0x3?, 0x2022a4d?)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac6993960 sp=0x170ac6993888 pc=0x38dd40b\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194700?}, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x170ac6764a48, 0x3, 0x4})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69939e0 sp=0x170ac6993960 pc=0x38e51e2\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194700}, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x0?, 0x2?, 0x1?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac6993a78 sp=0x170ac69939e0 pc=0x38e4eaf\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020500}, 0x170ac65fa000)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac6993d60 sp=0x170ac6993a78 pc=0x391b5c5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac5fe1880?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020500?}, 0x0?)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac6993e88 sp=0x170ac6993d60 pc=0x3929c16\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac6993f78 sp=0x170ac6993e88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac6993fe0 sp=0x170ac6993f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6993fe8 sp=0x170ac6993fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 101 gp=0x170ac61d4000 m=nil [sync.WaitGroup.Wait]:\n    runtime.gopark(0x170ac5de7e00?, 0x1ed0765?, 0xb0?, 0xfa?, 0xe?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de7da0 sp=0x170ac5de7d80 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x170ac65f8290, 0x0, 0x1, 0x0, 0x19)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x170ac5de7e08 sp=0x170ac5de7da0 pc=0x1f162b2\n    sync.runtime_SemacquireWaitGroup(0x170ac6709a70?, 0x0?)\n    \t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x170ac5de7e40 sp=0x170ac5de7e08 pc=0x1f3a66e\n    sync.(*WaitGroup).Wait(0x170ac65f8288)\n    \t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x170ac5de7e68 sp=0x170ac5de7e40 pc=0x1f4f445\n    golang.org/x/sync/errgroup.(*Group).Wait(0x170ac65f8280)\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x170ac5de7e88 sp=0x170ac5de7e68 pc=0x362507e\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x170ac5fe1140, {0x59f0b40, 0x170ac5c49130}, {0x170ac626a1e0, 0x6, 0x170ac5cac1c0?}, 0x170ac65fa000)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x170ac5de7f00 sp=0x170ac5de7e88 pc=0x392976a\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x170ac5de7f78 sp=0x170ac5de7f00 pc=0x392808e\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5de7fe0 sp=0x170ac5de7f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de7fe8 sp=0x170ac5de7fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 67 gp=0x170ac62f5a40 m=nil [IO wait]:\n    runtime.gopark(0x170ac6497780?, 0x8?, 0x40?, 0x16?, 0xb?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de47b8 sp=0x170ac5de4798 pc=0x1f38d2e\n    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n    \t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x170ac5de47f0 sp=0x170ac5de47b8 pc=0x1ef9e57\n    internal/poll.runtime_pollWait(0x7f7eb1670000, 0x72)\n    \t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x170ac5de4810 sp=0x170ac5de47f0 pc=0x1f375c5\n    internal/poll.(*pollDesc).wait(0x170ac620b480?, 0x170ac65a2000?, 0x0)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x170ac5de4838 sp=0x170ac5de4810 pc=0x1fbd167\n    internal/poll.(*pollDesc).waitRead(...)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\n    internal/poll.(*FD).Read(0x170ac620b480, {0x170ac65a2000, 0x3500, 0x3500})\n    \t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x170ac5de48d0 sp=0x170ac5de4838 pc=0x1fbe4ce\n    net.(*netFD).Read(0x170ac620b480, {0x170ac65a2000?, 0x170ac5de4901?, 0x22a2e75?})\n    \t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x170ac5de4918 sp=0x170ac5de48d0 pc=0x20798c5\n    net.(*conn).Read(0x170ac62823d8, {0x170ac65a2000?, 0x7f7eb047fc00?, 0x7f7ef9af2e00?})\n    \t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x170ac5de4960 sp=0x170ac5de4918 pc=0x208b7a5\n    crypto/tls.(*atLeastReader).Read(0x170ac6708f30, {0x170ac65a2000?, 0x170ac5de4a48?, 0x1efcc28?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x170ac5de49a8 sp=0x170ac5de4960 pc=0x2222ddb\n    bytes.(*Buffer).ReadFrom(0x170ac605fb28, {0x59a3680, 0x170ac6708f30})\n    \t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x170ac5de4a00 sp=0x170ac5de49a8 pc=0x1fe9258\n    crypto/tls.(*Conn).readFromUntil(0x170ac605f888, {0x59a32c0, 0x170ac62823d8}, 0x222764c?)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x170ac5de4a38 sp=0x170ac5de4a00 pc=0x2222fbe\n    crypto/tls.(*Conn).readRecordOrCCS(0x170ac605f888, 0x0)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x170ac5de4c18 sp=0x170ac5de4a38 pc=0x222009b\n    crypto/tls.(*Conn).readRecord(...)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:588\n    crypto/tls.(*Conn).Read(0x170ac605f888, {0x170ac6029000, 0x1000, 0x22c34e0?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x170ac5de4c80 sp=0x170ac5de4c18 pc=0x22267c5\n    bufio.(*Reader).Read(0x170ac647aa20, {0x170ac6153624, 0x9, 0x22e100e?})\n    \t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0x170ac5de4cb8 sp=0x170ac5de4c80 pc=0x20a71f7\n    io.ReadAtLeast({0x59a1260, 0x170ac647aa20}, {0x170ac6153624, 0x9, 0x9}, 0x9)\n    \t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0x170ac5de4d00 sp=0x170ac5de4cb8 pc=0x1fb17ee\n    io.ReadFull(...)\n    \t/usr/lib/golang/src/io/io.go:354\n    net/http.http2readFrameHeader({0x170ac6153624, 0x9, 0x170a00000005?}, {0x59a1260?, 0x170ac647aa20?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:1805 +0x65 fp=0x170ac5de4d50 sp=0x170ac5de4d00 pc=0x22b9e25\n    net/http.(*http2Framer).ReadFrameHeader(0x170ac61535e0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:2071 +0x6b fp=0x170ac5de4dd8 sp=0x170ac5de4d50 pc=0x22ba9eb\n    net/http.(*http2Framer).ReadFrame(0x170ac61535e0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:2130 +0x18 fp=0x170ac5de4e18 sp=0x170ac5de4dd8 pc=0x22bb1b8\n    net/http.(*http2clientConnReadLoop).run(0x170ac5de4fa8)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9550 +0xca fp=0x170ac5de4f60 sp=0x170ac5de4e18 pc=0x22de14a\n    net/http.(*http2ClientConn).readLoop(0x170ac61d9880)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x170ac5de4fc8 sp=0x170ac5de4f60 pc=0x22dd532\n    net/http.(*http2Transport).newClientConn.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x170ac5de4fe0 sp=0x170ac5de4fc8 pc=0x22d7057\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de4fe8 sp=0x170ac5de4fe0 pc=0x1f40801\n    created by net/http.(*http2Transport).newClientConn in goroutine 66\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5\n    \n    goroutine 73 gp=0x170ac648ad20 m=nil [IO wait]:\n    runtime.gopark(0x170ac6497e40?, 0x1edb176?, 0x80?, 0xed?, 0xb?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de6808 sp=0x170ac5de67e8 pc=0x1f38d2e\n    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n    \t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x170ac5de6840 sp=0x170ac5de6808 pc=0x1ef9e57\n    internal/poll.runtime_pollWait(0x7f7eb166fe00, 0x72)\n    \t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x170ac5de6860 sp=0x170ac5de6840 pc=0x1f375c5\n    internal/poll.(*pollDesc).wait(0x170ac60c4e80?, 0x170ac5ecd300?, 0x0)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x170ac5de6888 sp=0x170ac5de6860 pc=0x1fbd167\n    internal/poll.(*pollDesc).waitRead(...)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\n    internal/poll.(*FD).Read(0x170ac60c4e80, {0x170ac5ecd300, 0x1300, 0x1300})\n    \t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x170ac5de6920 sp=0x170ac5de6888 pc=0x1fbe4ce\n    net.(*netFD).Read(0x170ac60c4e80, {0x170ac5ecd300?, 0x170ac5ecd30a?, 0x5?})\n    \t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x170ac5de6968 sp=0x170ac5de6920 pc=0x20798c5\n    net.(*conn).Read(0x170ac5d82278, {0x170ac5ecd300?, 0x7f7eb047fc00?, 0x7f7ef9af2e00?})\n    \t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x170ac5de69b0 sp=0x170ac5de6968 pc=0x208b7a5\n    crypto/tls.(*atLeastReader).Read(0x170ac67088e8, {0x170ac5ecd300?, 0x170ac5de6a98?, 0x1efcc28?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x170ac5de69f8 sp=0x170ac5de69b0 pc=0x2222ddb\n    bytes.(*Buffer).ReadFrom(0x170ac5dd02a8, {0x59a3680, 0x170ac67088e8})\n    \t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x170ac5de6a50 sp=0x170ac5de69f8 pc=0x1fe9258\n    crypto/tls.(*Conn).readFromUntil(0x170ac5dd0008, {0x59a32c0, 0x170ac5d82278}, 0x222764c?)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x170ac5de6a88 sp=0x170ac5de6a50 pc=0x2222fbe\n    crypto/tls.(*Conn).readRecordOrCCS(0x170ac5dd0008, 0x0)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x170ac5de6c68 sp=0x170ac5de6a88 pc=0x222009b\n    crypto/tls.(*Conn).readRecord(...)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:588\n    crypto/tls.(*Conn).Read(0x170ac5dd0008, {0x170ac6564000, 0x1000, 0x4e55540?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x170ac5de6cd0 sp=0x170ac5de6c68 pc=0x22267c5\n    net/http.(*persistConn).Read(0x170ac5f73180, {0x170ac6564000?, 0x59a0ea0?, 0x81260e0?})\n    \t/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x170ac5de6d30 sp=0x170ac5de6cd0 pc=0x2319d07\n    bufio.(*Reader).fill(0x170ac65537a0)\n    \t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x170ac5de6d68 sp=0x170ac5de6d30 pc=0x20a6c63\n    bufio.(*Reader).Peek(0x170ac65537a0, 0x1)\n    \t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x170ac5de6d80 sp=0x170ac5de6d68 pc=0x20a6df3\n    net/http.(*persistConn).readLoop(0x170ac5f73180)\n    \t/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x170ac5de6fc8 sp=0x170ac5de6d80 pc=0x231a812\n    net/http.(*Transport).dialConn.gowrap2()\n    \t/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x170ac5de6fe0 sp=0x170ac5de6fc8 pc=0x2319197\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de6fe8 sp=0x170ac5de6fe0 pc=0x1f40801\n    created by net/http.(*Transport).dialConn in goroutine 82\n    \t/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac\n    \n    goroutine 103 gp=0x170ac648af00 m=nil [select]:\n    runtime.gopark(0x170ac5de0d00?, 0x2?, 0xa0?, 0x6a?, 0x170ac5de0c74?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de0ac8 sp=0x170ac5de0aa8 pc=0x1f38d2e\n    runtime.selectgo(0x170ac5de0d00, 0x170ac5de0c70, 0x553bfdc?, 0x0, 0x5a48c80?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5de0bf8 sp=0x170ac5de0ac8 pc=0x1f152a5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x0?, {0x59f0b40, 0x170ac5c49130}, 0x170ac65fa000, 0x170ac65fa070)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x170ac5de0f10 sp=0x170ac5de0bf8 pc=0x392836e\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x170ac5de0f78 sp=0x170ac5de0f10 pc=0x3927f3d\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5de0fe0 sp=0x170ac5de0f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de0fe8 sp=0x170ac5de0fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 74 gp=0x170ac648b4a0 m=nil [select]:\n    runtime.gopark(0x170ac5df1f48?, 0x2?, 0x0?, 0xb6?, 0x170ac5df1ef4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6760d88 sp=0x170ac6760d68 pc=0x1f38d2e\n    runtime.selectgo(0x170ac6760f48, 0x170ac5df1ef0, 0x170ac68d6980?, 0x0, 0x170ac655e8d0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6760eb8 sp=0x170ac6760d88 pc=0x1f152a5\n    net/http.(*persistConn).writeLoop(0x170ac5f73180)\n    \t/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x170ac6760fc8 sp=0x170ac6760eb8 pc=0x231c426\n    net/http.(*Transport).dialConn.gowrap3()\n    \t/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x170ac6760fe0 sp=0x170ac6760fc8 pc=0x2319157\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6760fe8 sp=0x170ac6760fe0 pc=0x1f40801\n    created by net/http.(*Transport).dialConn in goroutine 82\n    \t/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe\n    \n    goroutine 105 gp=0x170ac61d43c0 m=nil [select]:\n    runtime.gopark(0x170ac699cac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac699c9d8?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac699c848 sp=0x170ac699c828 pc=0x1f38d2e\n    runtime.selectgo(0x170ac699cac8, 0x170ac699c9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac699c978 sp=0x170ac699c848 pc=0x1f152a5\n    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e000, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac699cb40 sp=0x170ac699c978 pc=0x22d92e5\n    net/http.(*http2ClientConn).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8603\n    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e000, {0x4c?, 0xfa?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac699cc80 sp=0x170ac699cb40 pc=0x22d50ad\n    net/http.(*http2Transport).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7852\n    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e000?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac699ccc0 sp=0x170ac699cc80 pc=0x22e3a9c\n    net/http.(*Transport).roundTrip(0x8175400, 0x170ac619fcc0)\n    \t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac699cea0 sp=0x170ac699ccc0 pc=0x231037f\n    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n    \t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac699cec0 sp=0x170ac699cea0 pc=0x2321178\n    net/http.send(0x170ac619fcc0, {0x59a1ce0, 0x8175400}, {0x170ac699d138?, 0x1f3bac6?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac699d0b0 sp=0x170ac699cec0 pc=0x22ad9ab\n    net/http.(*Client).send(0x170ac6410c60, 0x170ac619fcc0, {0x170ac699d1e8?, 0x1fdd787?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac699d148 sp=0x170ac699d0b0 pc=0x22ad1d8\n    net/http.(*Client).do(0x170ac6410c60, 0x170ac619fcc0)\n    \t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac699d338 sp=0x170ac699d148 pc=0x22af817\n    net/http.(*Client).Do(...)\n    \t/usr/lib/golang/src/net/http/client.go:592\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e000}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e36b0}, 0x170ac65f7e00, {0x170ac65d9680, 0x118})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac699d630 sp=0x170ac699d338 pc=0x38de6bd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e000}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x170ac65f7da0, 0x0, {0x0, ...}, ...})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac699d888 sp=0x170ac699d630 pc=0x38ddecd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e000?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de1a48?}, 0x3?, 0x2022a4d?)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac699d960 sp=0x170ac699d888 pc=0x38dd40b\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194850?}, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x170ac5de1a48, 0x3, 0x4})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac699d9e0 sp=0x170ac699d960 pc=0x38e51e2\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194850}, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x0?, 0x2?, 0x0?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac699da78 sp=0x170ac699d9e0 pc=0x38e4eaf\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac60207c0}, 0x170ac65fa000)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac699dd60 sp=0x170ac699da78 pc=0x391b5c5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac60207c0?}, 0x0?)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac699de88 sp=0x170ac699dd60 pc=0x3929c16\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac699df78 sp=0x170ac699de88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac699dfe0 sp=0x170ac699df78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac699dfe8 sp=0x170ac699dfe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 107 gp=0x170ac61d45a0 m=nil [select]:\n    runtime.gopark(0x170ac69b2ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69b29d8?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69b2848 sp=0x170ac69b2828 pc=0x1f38d2e\n    runtime.selectgo(0x170ac69b2ac8, 0x170ac69b29d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69b2978 sp=0x170ac69b2848 pc=0x1f152a5\n    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e500, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69b2b40 sp=0x170ac69b2978 pc=0x22d92e5\n    net/http.(*http2ClientConn).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8603\n    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e500, {0x4c?, 0xfa?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69b2c80 sp=0x170ac69b2b40 pc=0x22d50ad\n    net/http.(*http2Transport).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7852\n    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e500?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69b2cc0 sp=0x170ac69b2c80 pc=0x22e3a9c\n    net/http.(*Transport).roundTrip(0x8175400, 0x170ac699e3c0)\n    \t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69b2ea0 sp=0x170ac69b2cc0 pc=0x231037f\n    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n    \t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69b2ec0 sp=0x170ac69b2ea0 pc=0x2321178\n    net/http.send(0x170ac699e3c0, {0x59a1ce0, 0x8175400}, {0x170ac69b3138?, 0x1f3bac6?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69b30b0 sp=0x170ac69b2ec0 pc=0x22ad9ab\n    net/http.(*Client).send(0x170ac6410c60, 0x170ac699e3c0, {0x4d3aa8?, 0xcd6610?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69b3148 sp=0x170ac69b30b0 pc=0x22ad1d8\n    net/http.(*Client).do(0x170ac6410c60, 0x170ac699e3c0)\n    \t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69b3338 sp=0x170ac69b3148 pc=0x22af817\n    net/http.(*Client).Do(...)\n    \t/usr/lib/golang/src/net/http/client.go:592\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e1e0}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3a60}, 0x170ac69a0120, {0x170ac65d98c0, 0x111})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69b3630 sp=0x170ac69b3338 pc=0x38de6bd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e1e0}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x170ac69a00c0, 0x0, {0x0, ...}, ...})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69b3888 sp=0x170ac69b3630 pc=0x38ddecd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e1e0?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de2a48?}, 0x3?, 0x2022a4d?)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69b3960 sp=0x170ac69b3888 pc=0x38dd40b\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194bd0?}, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x170ac5de2a48, 0x3, 0x4})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69b39e0 sp=0x170ac69b3960 pc=0x38e51e2\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194bd0}, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x0?, 0x2?, 0x0?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69b3a78 sp=0x170ac69b39e0 pc=0x38e4eaf\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020ec0}, 0x170ac65fa000)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69b3d60 sp=0x170ac69b3a78 pc=0x391b5c5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020ec0?}, 0x0?)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69b3e88 sp=0x170ac69b3d60 pc=0x3929c16\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69b3f78 sp=0x170ac69b3e88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69b3fe0 sp=0x170ac69b3f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69b3fe8 sp=0x170ac69b3fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 106 gp=0x170ac648b680 m=nil [select]:\n    runtime.gopark(0x170ac69aaac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69aa9d8?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69aa848 sp=0x170ac69aa828 pc=0x1f38d2e\n    runtime.selectgo(0x170ac69aaac8, 0x170ac69aa9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69aa978 sp=0x170ac69aa848 pc=0x1f152a5\n    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e280, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69aab40 sp=0x170ac69aa978 pc=0x22d92e5\n    net/http.(*http2ClientConn).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8603\n    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e280, {0x4c?, 0xfa?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69aac80 sp=0x170ac69aab40 pc=0x22d50ad\n    net/http.(*http2Transport).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7852\n    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e280?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69aacc0 sp=0x170ac69aac80 pc=0x22e3a9c\n    net/http.(*Transport).roundTrip(0x8175400, 0x170ac699e140)\n    \t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69aaea0 sp=0x170ac69aacc0 pc=0x231037f\n    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n    \t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69aaec0 sp=0x170ac69aaea0 pc=0x2321178\n    net/http.send(0x170ac699e140, {0x59a1ce0, 0x8175400}, {0x170ac69ab138?, 0x1f3bac6?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69ab0b0 sp=0x170ac69aaec0 pc=0x22ad9ab\n    net/http.(*Client).send(0x170ac6410c60, 0x170ac699e140, {0x170ac69ab1e8?, 0x1fdd787?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69ab148 sp=0x170ac69ab0b0 pc=0x22ad1d8\n    net/http.(*Client).do(0x170ac6410c60, 0x170ac699e140)\n    \t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69ab338 sp=0x170ac69ab148 pc=0x22af817\n    net/http.(*Client).Do(...)\n    \t/usr/lib/golang/src/net/http/client.go:592\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e0f0}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e38c0}, 0x170ac69a0030, {0x170ac65d97a0, 0x10e})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69ab630 sp=0x170ac69ab338 pc=0x38de6bd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e0f0}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x170ac65f7ec0, 0x0, {0x0, ...}, ...})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69ab888 sp=0x170ac69ab630 pc=0x38ddecd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e0f0?}, 0xa8?, {0x170ac5db1a50?, 0x170ac6766a48?}, 0x3?, 0x2022a4d?)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69ab960 sp=0x170ac69ab888 pc=0x38dd40b\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194a80?}, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x170ac6766a48, 0x3, 0x4})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69ab9e0 sp=0x170ac69ab960 pc=0x38e51e2\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194a80}, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x0?, 0x2?, 0x0?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69aba78 sp=0x170ac69ab9e0 pc=0x38e4eaf\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020cc0}, 0x170ac65fa000)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69abd60 sp=0x170ac69aba78 pc=0x391b5c5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac6766ea0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020cc0?}, 0x170ac5f6f3b0?)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69abe88 sp=0x170ac69abd60 pc=0x3929c16\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69abf78 sp=0x170ac69abe88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69abfe0 sp=0x170ac69abf78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69abfe8 sp=0x170ac69abfe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 108 gp=0x170ac61d4780 m=nil [select]:\n    runtime.gopark(0x170ac69c0ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69c09d8?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69c0848 sp=0x170ac69c0828 pc=0x1f38d2e\n    runtime.selectgo(0x170ac69c0ac8, 0x170ac69c09d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69c0978 sp=0x170ac69c0848 pc=0x1f152a5\n    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e780, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69c0b40 sp=0x170ac69c0978 pc=0x22d92e5\n    net/http.(*http2ClientConn).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8603\n    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e780, {0x4c?, 0xfa?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69c0c80 sp=0x170ac69c0b40 pc=0x22d50ad\n    net/http.(*http2Transport).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7852\n    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e780?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69c0cc0 sp=0x170ac69c0c80 pc=0x22e3a9c\n    net/http.(*Transport).roundTrip(0x8175400, 0x170ac699e640)\n    \t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69c0ea0 sp=0x170ac69c0cc0 pc=0x231037f\n    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n    \t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69c0ec0 sp=0x170ac69c0ea0 pc=0x2321178\n    net/http.send(0x170ac699e640, {0x59a1ce0, 0x8175400}, {0x170ac69c1138?, 0x1f3bac6?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69c10b0 sp=0x170ac69c0ec0 pc=0x22ad9ab\n    net/http.(*Client).send(0x170ac6410c60, 0x170ac699e640, {0x82bed80?, 0x6401fdd700?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69c1148 sp=0x170ac69c10b0 pc=0x22ad1d8\n    net/http.(*Client).do(0x170ac6410c60, 0x170ac699e640)\n    \t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69c1338 sp=0x170ac69c1148 pc=0x22af817\n    net/http.(*Client).Do(...)\n    \t/usr/lib/golang/src/net/http/client.go:592\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e280}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3bf0}, 0x170ac69a0210, {0x170ac65d99e0, 0x110})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69c1630 sp=0x170ac69c1338 pc=0x38de6bd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e280}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x170ac69a01b0, 0x0, {0x0, ...}, ...})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69c1888 sp=0x170ac69c1630 pc=0x38ddecd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e280?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de2a48?}, 0x3?, 0x2022a4d?)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69c1960 sp=0x170ac69c1888 pc=0x38dd40b\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194d90?}, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x170ac5de2a48, 0x3, 0x4})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69c19e0 sp=0x170ac69c1960 pc=0x38e51e2\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194d90}, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x0?, 0x2?, 0x0?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69c1a78 sp=0x170ac69c19e0 pc=0x38e4eaf\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6021d80}, 0x170ac65fa000)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69c1d60 sp=0x170ac69c1a78 pc=0x391b5c5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac66e1d6a?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6021d80?}, 0x0?)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69c1e88 sp=0x170ac69c1d60 pc=0x3929c16\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69c1f78 sp=0x170ac69c1e88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69c1fe0 sp=0x170ac69c1f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69c1fe8 sp=0x170ac69c1fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 109 gp=0x170ac61d4960 m=nil [select]:\n    runtime.gopark(0x170ac69caac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69ca9d8?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69ca848 sp=0x170ac69ca828 pc=0x1f38d2e\n    runtime.selectgo(0x170ac69caac8, 0x170ac69ca9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69ca978 sp=0x170ac69ca848 pc=0x1f152a5\n    net/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699ea00, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69cab40 sp=0x170ac69ca978 pc=0x22d92e5\n    net/http.(*http2ClientConn).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8603\n    net/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699ea00, {0x4c?, 0xfa?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69cac80 sp=0x170ac69cab40 pc=0x22d50ad\n    net/http.(*http2Transport).RoundTrip(...)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:7852\n    net/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699ea00?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69cacc0 sp=0x170ac69cac80 pc=0x22e3a9c\n    net/http.(*Transport).roundTrip(0x8175400, 0x170ac699e8c0)\n    \t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69caea0 sp=0x170ac69cacc0 pc=0x231037f\n    net/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\n    \t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69caec0 sp=0x170ac69caea0 pc=0x2321178\n    net/http.send(0x170ac699e8c0, {0x59a1ce0, 0x8175400}, {0x170ac69cb138?, 0x1f3bac6?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69cb0b0 sp=0x170ac69caec0 pc=0x22ad9ab\n    net/http.(*Client).send(0x170ac6410c60, 0x170ac699e8c0, {0x170ac69cb1e8?, 0x1fdd787?, 0x0?})\n    \t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69cb148 sp=0x170ac69cb0b0 pc=0x22ad1d8\n    net/http.(*Client).do(0x170ac6410c60, 0x170ac699e8c0)\n    \t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69cb338 sp=0x170ac69cb148 pc=0x22af817\n    net/http.(*Client).Do(...)\n    \t/usr/lib/golang/src/net/http/client.go:592\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e410}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3d80}, 0x170ac69a0300, {0x170ac65d9b00, 0x114})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69cb630 sp=0x170ac69cb338 pc=0x38de6bd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e410}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x170ac69a02a0, 0x0, {0x0, ...}, ...})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69cb888 sp=0x170ac69cb630 pc=0x38ddecd\n    github.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e410?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de3a48?}, 0x3?, 0x2022a4d?)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69cb960 sp=0x170ac69cb888 pc=0x38dd40b\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194620?}, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x170ac5de3a48, 0x3, 0x4})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69cb9e0 sp=0x170ac69cb960 pc=0x38e51e2\n    github.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194620}, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x0?, 0x2?, 0x2700000000000000?})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69cba78 sp=0x170ac69cb9e0 pc=0x38e4eaf\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac65f81c0}, 0x170ac65fa000)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69cbd60 sp=0x170ac69cba78 pc=0x391b5c5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac65f81c0?}, 0x0?)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69cbe88 sp=0x170ac69cbd60 pc=0x3929c16\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69cbf78 sp=0x170ac69cbe88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69cbfe0 sp=0x170ac69cbf78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69cbfe8 sp=0x170ac69cbfe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 110 gp=0x170ac61d4b40 m=nil [select]:\n    runtime.gopark(0x170ac6927f08?, 0x6?, 0x40?, 0x4b?, 0x170ac6927da4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6927c08 sp=0x170ac6927be8 pc=0x1f38d2e\n    runtime.selectgo(0x170ac6927f08, 0x170ac6927d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6927d38 sp=0x170ac6927c08 pc=0x1f152a5\n    net/http.(*http2clientStream).writeRequest(0x170ac61a0f00, 0x170ac619fb80, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6927f90 sp=0x170ac6927d38 pc=0x22da6a8\n    net/http.(*http2clientStream).doRequest(0x170ac61a0f00, 0x0?, 0x170ac66e2798?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6927fb8 sp=0x170ac6927f90 pc=0x22d99d8\n    net/http.(*http2ClientConn).roundTrip.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6927fe0 sp=0x170ac6927fb8 pc=0x22d999f\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6927fe8 sp=0x170ac6927fe0 pc=0x1f40801\n    created by net/http.(*http2ClientConn).roundTrip in goroutine 104\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n    \n    goroutine 111 gp=0x170ac61d4d20 m=nil [select]:\n    runtime.gopark(0x170ac658df08?, 0x6?, 0x0?, 0x7?, 0x170ac658dda4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac658dc08 sp=0x170ac658dbe8 pc=0x1f38d2e\n    runtime.selectgo(0x170ac658df08, 0x170ac658dd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac658dd38 sp=0x170ac658dc08 pc=0x1f152a5\n    net/http.(*http2clientStream).writeRequest(0x170ac61a1080, 0x170ac699e000, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac658df90 sp=0x170ac658dd38 pc=0x22da6a8\n    net/http.(*http2clientStream).doRequest(0x170ac61a1080, 0x0?, 0x170ac66e2798?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac658dfb8 sp=0x170ac658df90 pc=0x22d99d8\n    net/http.(*http2ClientConn).roundTrip.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac658dfe0 sp=0x170ac658dfb8 pc=0x22d999f\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac658dfe8 sp=0x170ac658dfe0 pc=0x1f40801\n    created by net/http.(*http2ClientConn).roundTrip in goroutine 105\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n    \n    goroutine 112 gp=0x170ac61d4f00 m=nil [select]:\n    runtime.gopark(0x170ac6589f08?, 0x6?, 0x0?, 0x7?, 0x170ac6589da4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6589c08 sp=0x170ac6589be8 pc=0x1f38d2e\n    runtime.selectgo(0x170ac6589f08, 0x170ac6589d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6589d38 sp=0x170ac6589c08 pc=0x1f152a5\n    net/http.(*http2clientStream).writeRequest(0x170ac61a1200, 0x170ac699e280, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6589f90 sp=0x170ac6589d38 pc=0x22da6a8\n    net/http.(*http2clientStream).doRequest(0x170ac61a1200, 0x0?, 0x170ac66e1f98?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6589fb8 sp=0x170ac6589f90 pc=0x22d99d8\n    net/http.(*http2ClientConn).roundTrip.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6589fe0 sp=0x170ac6589fb8 pc=0x22d999f\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6589fe8 sp=0x170ac6589fe0 pc=0x1f40801\n    created by net/http.(*http2ClientConn).roundTrip in goroutine 106\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n    \n    goroutine 113 gp=0x170ac61d50e0 m=nil [select]:\n    runtime.gopark(0x170ac658ff08?, 0x6?, 0x0?, 0x7?, 0x170ac658fda4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac658fc08 sp=0x170ac658fbe8 pc=0x1f38d2e\n    runtime.selectgo(0x170ac658ff08, 0x170ac658fd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac658fd38 sp=0x170ac658fc08 pc=0x1f152a5\n    net/http.(*http2clientStream).writeRequest(0x170ac61a1380, 0x170ac699e780, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac658ff90 sp=0x170ac658fd38 pc=0x22da6a8\n    net/http.(*http2clientStream).doRequest(0x170ac61a1380, 0x0?, 0x170ac66e1f98?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac658ffb8 sp=0x170ac658ff90 pc=0x22d99d8\n    net/http.(*http2ClientConn).roundTrip.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac658ffe0 sp=0x170ac658ffb8 pc=0x22d999f\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac658ffe8 sp=0x170ac658ffe0 pc=0x1f40801\n    created by net/http.(*http2ClientConn).roundTrip in goroutine 108\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n    \n    goroutine 114 gp=0x170ac61d52c0 m=nil [select]:\n    runtime.gopark(0x170ac6923f08?, 0x6?, 0x0?, 0x7?, 0x170ac6923da4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6923c08 sp=0x170ac6923be8 pc=0x1f38d2e\n    runtime.selectgo(0x170ac6923f08, 0x170ac6923d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6923d38 sp=0x170ac6923c08 pc=0x1f152a5\n    net/http.(*http2clientStream).writeRequest(0x170ac61a1500, 0x170ac699ea00, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6923f90 sp=0x170ac6923d38 pc=0x22da6a8\n    net/http.(*http2clientStream).doRequest(0x170ac61a1500, 0x0?, 0x170ac66e1f98?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6923fb8 sp=0x170ac6923f90 pc=0x22d99d8\n    net/http.(*http2ClientConn).roundTrip.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6923fe0 sp=0x170ac6923fb8 pc=0x22d999f\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6923fe8 sp=0x170ac6923fe0 pc=0x1f40801\n    created by net/http.(*http2ClientConn).roundTrip in goroutine 109\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n    \n    goroutine 115 gp=0x170ac61d54a0 m=nil [select]:\n    runtime.gopark(0x170ac6925f08?, 0x6?, 0x38?, 0x5b?, 0x170ac6925da4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6925c08 sp=0x170ac6925be8 pc=0x1f38d2e\n    runtime.selectgo(0x170ac6925f08, 0x170ac6925d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6925d38 sp=0x170ac6925c08 pc=0x1f152a5\n    net/http.(*http2clientStream).writeRequest(0x170ac61a1680, 0x170ac699e500, 0x0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6925f90 sp=0x170ac6925d38 pc=0x22da6a8\n    net/http.(*http2clientStream).doRequest(0x170ac61a1680, 0x0?, 0x170ac66e1f98?)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6925fb8 sp=0x170ac6925f90 pc=0x22d99d8\n    net/http.(*http2ClientConn).roundTrip.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6925fe0 sp=0x170ac6925fb8 pc=0x22d999f\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6925fe8 sp=0x170ac6925fe0 pc=0x1f40801\n    created by net/http.(*http2ClientConn).roundTrip in goroutine 107\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\n    \n    rax    0x0\n    rbx    0x7f7eabfff640\n    rcx    0x7f7ef9baa94c\n    rdx    0x6\n    rdi    0x10a2e\n    rsi    0x10a30\n    rbp    0x10a30\n    rsp    0x7f7eabffe790\n    r8     0x7f7eabffe860\n    r9     0x7f7ef9cd04e0\n    r10    0x8\n    r11    0x246\n    r12    0x6\n    r13    0x7f7eabffeb60\n    r14    0x170ac5c1f0e0\n    r15    0xffffffffffffffff\n    rip    0x7f7ef9baa94c\n    rflags 0x246\n    cs     0x33\n    fs     0x0\n    gs     0x0\n    \n      query-infra-both: command failed: exit status 2\n    output: runtime/cgo: pthread_create failed: Resource temporarily unavailable\n    SIGABRT: abort\n    PC=0x7f5e9c16994c m=2 sigcode=18446744073709551610\n    \n    goroutine 0 gp=0x280029a48b40 m=2 mp=0x280029abb008 [idle]:\n    runtime: g 0 gp=0x280029a48b40: unknown pc 0x7f5e9c16994c\n    stack: frame={sp:0x7f5e554a7740, fp:0x0} stack=[0x7f5e54ca81e0,0x7f5e554a7de0)\n                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n    00007f5e554a7640: 00000000 01eb64e5  00000000 00000000  .d..............\n                      ^ \u003cinternal/runtime/syscall/linux.EpollWait+0x45\u003e\n    00007f5e554a7650: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7660: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7670: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7680: 00007f5e 554a76d4  00007f5e 554a7d18  .vJU^....}JU^...\n    00007f5e554a7690: 00000000 01efa894  00000000 00000000  ................\n                      ^ \u003cruntime.netpoll+0xd4\u003e\n    00007f5e554a76a0: 00007f5e 554a76d4  00000000 00000000  .vJU^...........\n    00007f5e554a76b0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a76c0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a76d0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a76e0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a76f0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7700: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7710: 00000000 00000000  00007f5e 9c176fa9  .........o..^...\n    00007f5e554a7720: 00000000 00000000  00000000 00000011  ................\n    00007f5e554a7730: 00000000 00000130  00007f5e 9c16993e  0.......\u003e...^...\n    00007f5e554a7740: 00000000 00000120  00000000 00000000   ...............\n                      ^ SP\n    00007f5e554a7750: 00000000 00000013  00000000 00000000  ................\n    00007f5e554a7760: 00000000 00000000  00007f5e 9c1eccff  ............^...\n    00007f5e554a7770: 00007f5e 327fc910  00007f5e 9c1eccb1  ...2^.......^...\n    00007f5e554a7780: 00007f5e 327fc640  00007f5e 554a7a50  @..2^...PzJU^...\n    00007f5e554a7790: 00007f5e 554a78be  00007f5e 554a78bf  .xJU^....xJU^...\n    00007f5e554a77a0: 00007f5e 327fc640  00007f5e 9c1677f5  @..2^....w..^...\n    00007f5e554a77b0: 00007f5e 50000020  00000000 00000120   ..P^... .......\n    00007f5e554a77c0: 00000000 003d0f00  5a23f32c fe4c8400  ..=.......L.,.#Z\n    00007f5e554a77d0: 00007f5e 554a8640  00000000 00000006  @.JU^...........\n    00007f5e554a77e0: 00000000 0bac42a0  00007f5e 554a7b10  .B.......{JU^...\n    00007f5e554a77f0: 00002800 29a48b40  00007f5e 9c11c646  @..).(..F...^...\n    00007f5e554a7800: 00007f5e 9c2d9e90  00007f5e 9c1067f3  ..-.^....g..^...\n    00007f5e554a7810: 00000000 00000020  5a23f32c fe4c8400   .........L.,.#Z\n    00007f5e554a7820: 00007f5e 31ffc000  00007f5e 9c1dbac4  ...1^.......^...\n    00007f5e554a7830: 00000000 00000000  00000000 0000000d  ................\n    runtime: g 0 gp=0x280029a48b40: unknown pc 0x7f5e9c16994c\n    stack: frame={sp:0x7f5e554a7740, fp:0x0} stack=[0x7f5e54ca81e0,0x7f5e554a7de0)\n                       7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\n    00007f5e554a7640: 00000000 01eb64e5  00000000 00000000  .d..............\n                      ^ \u003cinternal/runtime/syscall/linux.EpollWait+0x45\u003e\n    00007f5e554a7650: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7660: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7670: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7680: 00007f5e 554a76d4  00007f5e 554a7d18  .vJU^....}JU^...\n    00007f5e554a7690: 00000000 01efa894  00000000 00000000  ................\n                      ^ \u003cruntime.netpoll+0xd4\u003e\n    00007f5e554a76a0: 00007f5e 554a76d4  00000000 00000000  .vJU^...........\n    00007f5e554a76b0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a76c0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a76d0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a76e0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a76f0: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7700: 00000000 00000000  00000000 00000000  ................\n    00007f5e554a7710: 00000000 00000000  00007f5e 9c176fa9  .........o..^...\n    00007f5e554a7720: 00000000 00000000  00000000 00000011  ................\n    00007f5e554a7730: 00000000 00000130  00007f5e 9c16993e  0.......\u003e...^...\n    00007f5e554a7740: 00000000 00000120  00000000 00000000   ...............\n                      ^ SP\n    00007f5e554a7750: 00000000 00000013  00000000 00000000  ................\n    00007f5e554a7760: 00000000 00000000  00007f5e 9c1eccff  ............^...\n    00007f5e554a7770: 00007f5e 327fc910  00007f5e 9c1eccb1  ...2^.......^...\n    00007f5e554a7780: 00007f5e 327fc640  00007f5e 554a7a50  @..2^...PzJU^...\n    00007f5e554a7790: 00007f5e 554a78be  00007f5e 554a78bf  .xJU^....xJU^...\n    00007f5e554a77a0: 00007f5e 327fc640  00007f5e 9c1677f5  @..2^....w..^...\n    00007f5e554a77b0: 00007f5e 50000020  00000000 00000120   ..P^... .......\n    00007f5e554a77c0: 00000000 003d0f00  5a23f32c fe4c8400  ..=.......L.,.#Z\n    00007f5e554a77d0: 00007f5e 554a8640  00000000 00000006  @.JU^...........\n    00007f5e554a77e0: 00000000 0bac42a0  00007f5e 554a7b10  .B.......{JU^...\n    00007f5e554a77f0: 00002800 29a48b40  00007f5e 9c11c646  @..).(..F...^...\n    00007f5e554a7800: 00007f5e 9c2d9e90  00007f5e 9c1067f3  ..-.^....g..^...\n    00007f5e554a7810: 00000000 00000020  5a23f32c fe4c8400   .........L.,.#Z\n    00007f5e554a7820: 00007f5e 31ffc000  00007f5e 9c1dbac4  ...1^.......^...\n    00007f5e554a7830: 00000000 00000000  00000000 0000000d  ................\n    \n    goroutine 1 gp=0x280029a481e0 m=nil [sync.WaitGroup.Wait]:\n    runtime.gopark(0x28002a5927b8?, 0x2022a4d?, 0x30?, 0xe9?, 0x28002a5927e8?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a5927a0 sp=0x28002a592780 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x280029ae34d0, 0x0, 0x1, 0x0, 0x19)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a592808 sp=0x28002a5927a0 pc=0x1f162b2\n    sync.runtime_SemacquireWaitGroup(0x28002a83b578?, 0xe0?)\n    \t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x28002a592840 sp=0x28002a592808 pc=0x1f3a66e\n    sync.(*WaitGroup).Wait(0x280029ae34c8)\n    \t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x28002a592868 sp=0x28002a592840 pc=0x1f4f445\n    golang.org/x/sync/errgroup.(*Group).Wait(0x280029ae34c0)\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x28002a592888 sp=0x28002a592868 pc=0x362507e\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0}, {0x28002a977b50, 0x1, 0x1})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x28002a592948 sp=0x28002a592888 pc=0x3927e05\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).gatherInfraLogs(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:587 +0x867 fp=0x28002a592d30 sp=0x28002a592948 pc=0x3927727\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:333 +0x533 fp=0x28002a593908 sp=0x28002a592d30 pc=0x3921bf3\n    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedInfraQueryOptions).RunInfra(0x28002a11e1e0, {0x59f0b08, 0x28002a1873b0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:83 +0x71f fp=0x28002a593b30 sp=0x28002a593908 pc=0x393887f\n    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawInfraQueryOptions).Run(0x0?, {0x59f0b08, 0x28002a1873b0})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_options.go:109 +0x54 fp=0x28002a593b58 sp=0x28002a593b30 pc=0x3939134\n    github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryInfraCommand.func1(0x28002a53d508?, {0x280029ca3b00?, 0x4?, 0x553b0e2?})\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:45 +0x28 fp=0x28002a593b80 sp=0x28002a593b58 pc=0x3938128\n    github.com/spf13/cobra.(*Command).execute(0x28002a53d508, {0x280029ab8120, 0xc, 0xc})\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x28002a593d28 sp=0x28002a593b80 pc=0x212e614\n    github.com/spf13/cobra.(*Command).ExecuteC(0x280029ec9b08)\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x28002a593e18 sp=0x28002a593d28 pc=0x212efa5\n    github.com/spf13/cobra.(*Command).Execute(...)\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071\n    github.com/spf13/cobra.(*Command).ExecuteContext(...)\n    \t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064\n    main.main()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x28002a593f48 sp=0x28002a593e18 pc=0x47ddd6a\n    runtime.main()\n    \t/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x28002a593fe0 sp=0x28002a593f48 pc=0x1f01975\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a593fe8 sp=0x28002a593fe0 pc=0x1f40801\n    \n    goroutine 2 gp=0x280029a48d20 m=nil [force gc (idle)]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b08fa8 sp=0x280029b08f88 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.forcegchelper()\n    \t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x280029b08fe0 sp=0x280029b08fa8 pc=0x1f01c93\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b08fe8 sp=0x280029b08fe0 pc=0x1f40801\n    created by runtime.init.7 in goroutine 1\n    \t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\n    \n    goroutine 3 gp=0x280029a492c0 m=nil [GC sweep wait]:\n    runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b09788 sp=0x280029b09768 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.bgsweep(0x280029b30000)\n    \t/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x280029b097c8 sp=0x280029b09788 pc=0x1ee9f11\n    runtime.gcenable.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x280029b097e0 sp=0x280029b097c8 pc=0x1edadd7\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b097e8 sp=0x280029b097e0 pc=0x1f40801\n    created by runtime.gcenable in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\n    \n    goroutine 4 gp=0x280029a494a0 m=nil [GC scavenge wait]:\n    runtime.gopark(0x10000?, 0x5981cd8?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b09f78 sp=0x280029b09f58 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.(*scavengerState).park(0x82979c0)\n    \t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x280029b09fa8 sp=0x280029b09f78 pc=0x1ee77e9\n    runtime.bgscavenge(0x280029b30000)\n    \t/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x280029b09fc8 sp=0x280029b09fa8 pc=0x1ee7d99\n    runtime.gcenable.gowrap2()\n    \t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x280029b09fe0 sp=0x280029b09fc8 pc=0x1edad97\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b09fe8 sp=0x280029b09fe0 pc=0x1f40801\n    created by runtime.gcenable in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\n    \n    goroutine 5 gp=0x280029a49a40 m=nil [GOMAXPROCS updater (idle)]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0a788 sp=0x280029b0a768 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.updateMaxProcsGoroutine()\n    \t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x280029b0a7e0 sp=0x280029b0a788 pc=0x1f10267\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0a7e8 sp=0x280029b0a7e0 pc=0x1f40801\n    created by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\n    \t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\n    \n    goroutine 6 gp=0x280029b481e0 m=nil [finalizer wait]:\n    runtime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x280029b08601?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b08620 sp=0x280029b08600 pc=0x1f38d2e\n    runtime.runFinalizers()\n    \t/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x280029b087e0 sp=0x280029b08620 pc=0x1ed9cc7\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b087e8 sp=0x280029b087e0 pc=0x1f40801\n    created by runtime.createfing in goroutine 1\n    \t/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d\n    \n    goroutine 7 gp=0x280029d3fc20 m=nil [cleanup wait]:\n    runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0af68 sp=0x280029b0af48 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.(*cleanupQueue).dequeue(0x82980e0)\n    \t/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x280029b0afa0 sp=0x280029b0af68 pc=0x1ed6074\n    runtime.runCleanups()\n    \t/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x280029b0afe0 sp=0x280029b0afa0 pc=0x1ed66e5\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0afe8 sp=0x280029b0afe0 pc=0x1f40801\n    created by runtime.(*cleanupQueue).createGs in goroutine 1\n    \t/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5\n    \n    goroutine 8 gp=0x280029ee6000 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125c86ae32d7?, 0x0?, 0x0?, 0x0?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0b740 sp=0x280029b0b720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x280029ad7ab0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x280029b0b7c8 sp=0x280029b0b740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b0b7e0 sp=0x280029b0b7c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0b7e8 sp=0x280029b0b7e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 18 gp=0x280029b843c0 m=nil [GC worker (idle)]:\n    runtime.gopark(0x125c662e5759?, 0x3?, 0xc2?, 0x10?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b04740 sp=0x280029b04720 pc=0x1f38d2e\n    runtime.gcBgMarkWorker(0x280029ad7ab0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x280029b047c8 sp=0x280029b04740 pc=0x1edd8ab\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b047e0 sp=0x280029b047c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b047e8 sp=0x280029b047e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 9 gp=0x280029ee61e0 m=nil [semacquire]:\n    runtime.gopark(0x6e2b22?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029b0bc96?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0be38 sp=0x280029b0be18 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029b0bea0 sp=0x280029b0be38 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029b0bf40 sp=0x280029b0bea0 pc=0x1edbb12\n    runtime.gcBgMarkWorker(0x280029ad7ab0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029b0bfc8 sp=0x280029b0bf40 pc=0x1edda88\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b0bfe0 sp=0x280029b0bfc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0bfe8 sp=0x280029b0bfe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 10 gp=0x280029ee63c0 m=nil [semacquire]:\n    runtime.gopark(0x1ec6f5e?, 0x1f3ec12?, 0x80?, 0x51?, 0x1ec70df?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eec638 sp=0x280029eec618 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eec6a0 sp=0x280029eec638 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eec740 sp=0x280029eec6a0 pc=0x1edbb12\n    runtime.gcBgMarkWorker(0x280029ad7ab0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eec7c8 sp=0x280029eec740 pc=0x1edda88\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eec7e0 sp=0x280029eec7c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eec7e8 sp=0x280029eec7e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 11 gp=0x280029ee65a0 m=nil [semacquire]:\n    runtime.gopark(0x54b522?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029eecc96?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eece38 sp=0x280029eece18 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eecea0 sp=0x280029eece38 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eecf40 sp=0x280029eecea0 pc=0x1edbb12\n    runtime.gcBgMarkWorker(0x280029ad7ab0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eecfc8 sp=0x280029eecf40 pc=0x1edda88\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eecfe0 sp=0x280029eecfc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eecfe8 sp=0x280029eecfe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 34 gp=0x280029f04000 m=nil [semacquire]:\n    runtime.gopark(0x2d7122?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029ee8496?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029ee8638 sp=0x280029ee8618 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029ee86a0 sp=0x280029ee8638 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029ee8740 sp=0x280029ee86a0 pc=0x1edbb12\n    runtime.gcBgMarkWorker(0x280029ad7ab0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029ee87c8 sp=0x280029ee8740 pc=0x1edda88\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029ee87e0 sp=0x280029ee87c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029ee87e8 sp=0x280029ee87e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 19 gp=0x280029b845a0 m=4 mp=0x280029b34008 [GC mark termination]:\n    runtime.systemstack_switch()\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x280029b04ea0 sp=0x280029b04e90 pc=0x1f3ebc8\n    runtime.forEachP(...)\n    \t/usr/lib/golang/src/runtime/proc.go:2112\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x280029b04f40 sp=0x280029b04ea0 pc=0x1edbc34\n    runtime.gcBgMarkWorker(0x280029ad7ab0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029b04fc8 sp=0x280029b04f40 pc=0x1edda88\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b04fe0 sp=0x280029b04fc8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b04fe8 sp=0x280029b04fe0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 12 gp=0x280029ee6780 m=nil [semacquire]:\n    runtime.gopark(0x3f9e22?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029eed496?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eed638 sp=0x280029eed618 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eed6a0 sp=0x280029eed638 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eed740 sp=0x280029eed6a0 pc=0x1edbb12\n    runtime.gcBgMarkWorker(0x280029ad7ab0)\n    \t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eed7c8 sp=0x280029eed740 pc=0x1edda88\n    runtime.gcBgMarkStartWorkers.gowrap1()\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eed7e0 sp=0x280029eed7c8 pc=0x1edd797\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eed7e8 sp=0x280029eed7e0 pc=0x1f40801\n    created by runtime.gcBgMarkStartWorkers in goroutine 1\n    \t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n    \n    goroutine 35 gp=0x280029ee74a0 m=nil [select, locked to thread]:\n    runtime.gopark(0x280029eef7a8?, 0x2?, 0x1?, 0x74?, 0x280029eef794?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eef628 sp=0x280029eef608 pc=0x1f38d2e\n    runtime.selectgo(0x280029eef7a8, 0x280029eef790, 0x0?, 0x0, 0x0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x280029eef758 sp=0x280029eef628 pc=0x1f152a5\n    runtime.ensureSigM.func1()\n    \t/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x280029eef7e0 sp=0x280029eef758 pc=0x1f328e8\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eef7e8 sp=0x280029eef7e0 pc=0x1f40801\n    created by runtime.ensureSigM in goroutine 1\n    \t/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5\n    \n    goroutine 50 gp=0x280029f84960 m=9 mp=0x28002a000008 [syscall]:\n    runtime.notetsleepg(0x82bdea0, 0xffffffffffffffff)\n    \t/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x280029ee9fa0 sp=0x280029ee9f78 pc=0x1ece509\n    os/signal.signal_recv()\n    \t/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x280029ee9fc0 sp=0x280029ee9fa0 pc=0x1f3b298\n    os/signal.loop()\n    \t/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x280029ee9fe0 sp=0x280029ee9fc0 pc=0x2030553\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029ee9fe8 sp=0x280029ee9fe0 pc=0x1f40801\n    created by os/signal.Notify.func1.1 in goroutine 1\n    \t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\n    \n    goroutine 51 gp=0x280029f84b40 m=nil [select]:\n    runtime.gopark(0x280029eea798?, 0x2?, 0x80?, 0xa3?, 0x280029eea78c?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eea628 sp=0x280029eea608 pc=0x1f38d2e\n    runtime.selectgo(0x280029eea798, 0x280029eea788, 0x0?, 0x0, 0x0?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x280029eea758 sp=0x280029eea628 pc=0x1f152a5\n    os/signal.NotifyContext.func1()\n    \t/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x280029eea7e0 sp=0x280029eea758 pc=0x202ff9c\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eea7e8 sp=0x280029eea7e0 pc=0x1f40801\n    created by os/signal.NotifyContext in goroutine 1\n    \t/usr/lib/golang/src/os/signal/signal.go:291 +0x158\n    \n    goroutine 137 gp=0x28002a60a000 m=nil [sync.WaitGroup.Wait]:\n    runtime.gopark(0x28002a603e00?, 0x1ed0765?, 0x80?, 0xea?, 0x7f5e54234a40?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a603da0 sp=0x28002a603d80 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x280029ae3b10, 0x0, 0x1, 0x0, 0x19)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a603e08 sp=0x28002a603da0 pc=0x1f162b2\n    sync.runtime_SemacquireWaitGroup(0x28002a83b590?, 0x0?)\n    \t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x28002a603e40 sp=0x28002a603e08 pc=0x1f3a66e\n    sync.(*WaitGroup).Wait(0x280029ae3b08)\n    \t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x28002a603e68 sp=0x28002a603e40 pc=0x1f4f445\n    golang.org/x/sync/errgroup.(*Group).Wait(0x280029ae3b00)\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x28002a603e88 sp=0x28002a603e68 pc=0x362507e\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x280029d80e80, {0x59f0b40, 0x280029beca00}, {0x28002a977b50, 0x1, 0x38d788c?}, 0x28002a830c40)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x28002a603f00 sp=0x28002a603e88 pc=0x392976a\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x28002a603f78 sp=0x28002a603f00 pc=0x392808e\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a603fe0 sp=0x28002a603f78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a603fe8 sp=0x28002a603fe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 38 gp=0x28002a60b0e0 m=nil [IO wait]:\n    runtime.gopark(0x28002a7c22c0?, 0x280029aee7f8?, 0x40?, 0x77?, 0xb?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b197b8 sp=0x280029b19798 pc=0x1f38d2e\n    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n    \t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x280029b197f0 sp=0x280029b197b8 pc=0x1ef9e57\n    internal/poll.runtime_pollWait(0x7f5e554b6c00, 0x72)\n    \t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x280029b19810 sp=0x280029b197f0 pc=0x1f375c5\n    internal/poll.(*pollDesc).wait(0x280029bf6400?, 0x28002a90c000?, 0x0)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x280029b19838 sp=0x280029b19810 pc=0x1fbd167\n    internal/poll.(*pollDesc).waitRead(...)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\n    internal/poll.(*FD).Read(0x280029bf6400, {0x28002a90c000, 0x3500, 0x3500})\n    \t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x280029b198d0 sp=0x280029b19838 pc=0x1fbe4ce\n    net.(*netFD).Read(0x280029bf6400, {0x28002a90c000?, 0x28002a90c000?, 0x5?})\n    \t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x280029b19918 sp=0x280029b198d0 pc=0x20798c5\n    net.(*conn).Read(0x280029a46378, {0x28002a90c000?, 0x7f5e554a9828?, 0x7f5e9c0b1e00?})\n    \t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x280029b19960 sp=0x280029b19918 pc=0x208b7a5\n    crypto/tls.(*atLeastReader).Read(0x28002aa26ba0, {0x28002a90c000?, 0x28002a0d8658?, 0x1efcc28?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x280029b199a8 sp=0x280029b19960 pc=0x2222ddb\n    bytes.(*Buffer).ReadFrom(0x28002a0d8628, {0x59a3680, 0x28002aa26ba0})\n    \t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x280029b19a00 sp=0x280029b199a8 pc=0x1fe9258\n    crypto/tls.(*Conn).readFromUntil(0x28002a0d8388, {0x59a32c0, 0x280029a46378}, 0x222764c?)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x280029b19a38 sp=0x280029b19a00 pc=0x2222fbe\n    crypto/tls.(*Conn).readRecordOrCCS(0x28002a0d8388, 0x0)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x280029b19c18 sp=0x280029b19a38 pc=0x222009b\n    crypto/tls.(*Conn).readRecord(...)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:588\n    crypto/tls.(*Conn).Read(0x28002a0d8388, {0x280029fed000, 0x1000, 0x22c34e0?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x280029b19c80 sp=0x280029b19c18 pc=0x22267c5\n    bufio.(*Reader).Read(0x28002a7b89c0, {0x28002a7c8584, 0x9, 0x22e100e?})\n    \t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0x280029b19cb8 sp=0x280029b19c80 pc=0x20a71f7\n    io.ReadAtLeast({0x59a1260, 0x28002a7b89c0}, {0x28002a7c8584, 0x9, 0x9}, 0x9)\n    \t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0x280029b19d00 sp=0x280029b19cb8 pc=0x1fb17ee\n    io.ReadFull(...)\n    \t/usr/lib/golang/src/io/io.go:354\n    net/http.http2readFrameHeader({0x28002a7c8584, 0x9, 0x280000000015?}, {0x59a1260?, 0x28002a7b89c0?})\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:1805 +0x65 fp=0x280029b19d50 sp=0x280029b19d00 pc=0x22b9e25\n    net/http.(*http2Framer).ReadFrameHeader(0x28002a7c8540)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:2071 +0x6b fp=0x280029b19dd8 sp=0x280029b19d50 pc=0x22ba9eb\n    net/http.(*http2Framer).ReadFrame(0x28002a7c8540)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:2130 +0x18 fp=0x280029b19e18 sp=0x280029b19dd8 pc=0x22bb1b8\n    net/http.(*http2clientConnReadLoop).run(0x280029b19fa8)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9550 +0xca fp=0x280029b19f60 sp=0x280029b19e18 pc=0x22de14a\n    net/http.(*http2ClientConn).readLoop(0x280029bc01c0)\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x280029b19fc8 sp=0x280029b19f60 pc=0x22dd532\n    net/http.(*http2Transport).newClientConn.gowrap1()\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x280029b19fe0 sp=0x280029b19fc8 pc=0x22d7057\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b19fe8 sp=0x280029b19fe0 pc=0x1f40801\n    created by net/http.(*http2Transport).newClientConn in goroutine 37\n    \t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5\n    \n    goroutine 69 gp=0x28002a60b4a0 m=nil [IO wait]:\n    runtime.gopark(0x28002a50c560?, 0x1edb176?, 0x80?, 0xed?, 0xb?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a606808 sp=0x28002a6067e8 pc=0x1f38d2e\n    runtime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\n    \t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x28002a606840 sp=0x28002a606808 pc=0x1ef9e57\n    internal/poll.runtime_pollWait(0x7f5e554b6a00, 0x72)\n    \t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x28002a606860 sp=0x28002a606840 pc=0x1f375c5\n    internal/poll.(*pollDesc).wait(0x280029bf6c80?, 0x280029a5a000?, 0x0)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x28002a606888 sp=0x28002a606860 pc=0x1fbd167\n    internal/poll.(*pollDesc).waitRead(...)\n    \t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\n    internal/poll.(*FD).Read(0x280029bf6c80, {0x280029a5a000, 0x2500, 0x2500})\n    \t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x28002a606920 sp=0x28002a606888 pc=0x1fbe4ce\n    net.(*netFD).Read(0x280029bf6c80, {0x280029a5a000?, 0x28002a60690a?, 0x1edb176?})\n    \t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x28002a606968 sp=0x28002a606920 pc=0x20798c5\n    net.(*conn).Read(0x280029a463c0, {0x280029a5a000?, 0x7f5e5423eae0?, 0x7f5e9c09dde0?})\n    \t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x28002a6069b0 sp=0x28002a606968 pc=0x208b7a5\n    crypto/tls.(*atLeastReader).Read(0x28002a628318, {0x280029a5a000?, 0x28002a606a98?, 0x1efcc28?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x28002a6069f8 sp=0x28002a6069b0 pc=0x2222ddb\n    bytes.(*Buffer).ReadFrom(0x280029ba9ea8, {0x59a3680, 0x28002a628318})\n    \t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x28002a606a50 sp=0x28002a6069f8 pc=0x1fe9258\n    crypto/tls.(*Conn).readFromUntil(0x280029ba9c08, {0x59a32c0, 0x280029a463c0}, 0x222764c?)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x28002a606a88 sp=0x28002a606a50 pc=0x2222fbe\n    crypto/tls.(*Conn).readRecordOrCCS(0x280029ba9c08, 0x0)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x28002a606c68 sp=0x28002a606a88 pc=0x222009b\n    crypto/tls.(*Conn).readRecord(...)\n    \t/usr/lib/golang/src/crypto/tls/conn.go:588\n    crypto/tls.(*Conn).Read(0x280029ba9c08, {0x28002a24a000, 0x1000, 0x4e55540?})\n    \t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x28002a606cd0 sp=0x28002a606c68 pc=0x22267c5\n    net/http.(*persistConn).Read(0x28002a0ad400, {0x28002a24a000?, 0x59a0ea0?, 0x81260e0?})\n    \t/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x28002a606d30 sp=0x28002a606cd0 pc=0x2319d07\n    bufio.(*Reader).fill(0x28002a8130e0)\n    \t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x28002a606d68 sp=0x28002a606d30 pc=0x20a6c63\n    bufio.(*Reader).Peek(0x28002a8130e0, 0x1)\n    \t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x28002a606d80 sp=0x28002a606d68 pc=0x20a6df3\n    net/http.(*persistConn).readLoop(0x28002a0ad400)\n    \t/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x28002a606fc8 sp=0x28002a606d80 pc=0x231a812\n    net/http.(*Transport).dialConn.gowrap2()\n    \t/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x28002a606fe0 sp=0x28002a606fc8 pc=0x2319197\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a606fe8 sp=0x28002a606fe0 pc=0x1f40801\n    created by net/http.(*Transport).dialConn in goroutine 55\n    \t/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac\n    \n    goroutine 70 gp=0x28002a60bc20 m=nil [select]:\n    runtime.gopark(0x28002a613f48?, 0x2?, 0x0?, 0xb6?, 0x28002a613ef4?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a708d88 sp=0x28002a708d68 pc=0x1f38d2e\n    runtime.selectgo(0x28002a708f48, 0x28002a613ef0, 0x280029c93a80?, 0x0, 0x28002a4f7c80?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a708eb8 sp=0x28002a708d88 pc=0x1f152a5\n    net/http.(*persistConn).writeLoop(0x28002a0ad400)\n    \t/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x28002a708fc8 sp=0x28002a708eb8 pc=0x231c426\n    net/http.(*Transport).dialConn.gowrap3()\n    \t/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x28002a708fe0 sp=0x28002a708fc8 pc=0x2319157\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a708fe8 sp=0x28002a708fe0 pc=0x1f40801\n    created by net/http.(*Transport).dialConn in goroutine 55\n    \t/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe\n    \n    goroutine 138 gp=0x28002a84a780 m=nil [semacquire]:\n    runtime.gopark(0x1?, 0x1f78fa5?, 0xe0?, 0x75?, 0x2?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a2ca910 sp=0x28002a2ca8f0 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a2ca978 sp=0x28002a2ca910 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x28002a2caa18 sp=0x28002a2ca978 pc=0x1edbb12\n    runtime.gcAssistAlloc(0x28002a84a780)\n    \t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x28002a2caaa8 sp=0x28002a2caa18 pc=0x1ee06b6\n    runtime.deductAssistCredit(0x28002a2cac70?)\n    \t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x28002a2caac8 sp=0x28002a2caaa8 pc=0x1ed10a6\n    runtime.mallocgc(0x10, 0x4af39e0, 0x1)\n    \t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x28002a2caaf8 sp=0x28002a2caac8 pc=0x1f35e07\n    runtime.convTstring({0x28002a6a7728, 0x14})\n    \t/usr/lib/golang/src/runtime/iface.go:431 +0x45 fp=0x28002a2cab20 sp=0x28002a2caaf8 pc=0x1f35865\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x280029beca00}, 0x28002a830cb0, 0x280029f636e0)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:297 +0x394 fp=0x28002a2caf48 sp=0x28002a2cab20 pc=0x3920674\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x28002a2caf78 sp=0x28002a2caf48 pc=0x392800c\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a2cafe0 sp=0x28002a2caf78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a2cafe8 sp=0x28002a2cafe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 139 gp=0x28002a84a960 m=nil [select]:\n    runtime.gopark(0x28002a4bdd00?, 0x2?, 0x1?, 0xc9?, 0x28002a4bdc74?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a4bdac8 sp=0x28002a4bdaa8 pc=0x1f38d2e\n    runtime.selectgo(0x28002a4bdd00, 0x28002a4bdc70, 0x28002a3e8a50?, 0x0, 0x280029f827c8?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a4bdbf8 sp=0x28002a4bdac8 pc=0x1f152a5\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x49424d554554534c?, {0x59f0b40, 0x280029beca00}, 0x28002a830c40, 0x28002a830cb0)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x28002a4bdf10 sp=0x28002a4bdbf8 pc=0x392836e\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x28002a4bdf78 sp=0x28002a4bdf10 pc=0x3927f3d\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a4bdfe0 sp=0x28002a4bdf78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a4bdfe8 sp=0x28002a4bdfe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 140 gp=0x28002a84ab40 m=nil [semacquire]:\n    runtime.gopark(0x1f6aa32?, 0x28002a34b418?, 0xc5?, 0xbd?, 0x0?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a34b3c0 sp=0x28002a34b3a0 pc=0x1f38d2e\n    runtime.goparkunlock(...)\n    \t/usr/lib/golang/src/runtime/proc.go:468\n    runtime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\n    \t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a34b428 sp=0x28002a34b3c0 pc=0x1f162b2\n    runtime.semacquire(...)\n    \t/usr/lib/golang/src/runtime/sema.go:143\n    runtime.gcMarkDone()\n    \t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x28002a34b4c8 sp=0x28002a34b428 pc=0x1edbb12\n    runtime.gcAssistAlloc(0x28002a84ab40)\n    \t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x28002a34b558 sp=0x28002a34b4c8 pc=0x1ee06b6\n    runtime.deductAssistCredit(0x20?)\n    \t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x28002a34b578 sp=0x28002a34b558 pc=0x1ed10a6\n    runtime.mallocgc(0x1000, 0x4af32a0, 0x1)\n    \t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x28002a34b5a8 sp=0x28002a34b578 pc=0x1f35e07\n    runtime.makeslice(0x28002a34b668?, 0x1f8?, 0x1ff?)\n    \t/usr/lib/golang/src/runtime/slice.go:117 +0x49 fp=0x28002a34b5d0 sp=0x28002a34b5a8 pc=0x1f3b489\n    bufio.NewWriterSize(...)\n    \t/usr/lib/golang/src/bufio/bufio.go:599\n    bufio.NewWriter(...)\n    \t/usr/lib/golang/src/bufio/bufio.go:608\n    encoding/csv.NewWriter(...)\n    \t/usr/lib/golang/src/encoding/csv/writer.go:42\n    github.com/Azure/azure-kusto-go/azkustodata/query.(*row).String(0x812ab90?)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/row_impl.go:100 +0x18e fp=0x28002a34b6e8 sp=0x28002a34b5d0 pc=0x38cf5ee\n    fmt.(*pp).handleMethods(0x28002a24c0d0, 0x2a34b8f0?)\n    \t/usr/lib/golang/src/fmt/print.go:674 +0x2f4 fp=0x28002a34b8a8 sp=0x28002a34b6e8 pc=0x1fdd1d4\n    fmt.(*pp).printArg(0x28002a24c0d0, {0x549a3a0, 0x28002a7c7c80}, 0x76)\n    \t/usr/lib/golang/src/fmt/print.go:757 +0x42e fp=0x28002a34b938 sp=0x28002a34b8a8 pc=0x1fdd9ee\n    fmt.(*pp).doPrintf(0x28002a24c0d0, {0x553a400, 0x2}, {0x28002a34bd08, 0x1, 0x1})\n    \t/usr/lib/golang/src/fmt/print.go:1075 +0x3ec fp=0x28002a34ba20 sp=0x28002a34b938 pc=0x1fe04ec\n    fmt.Sprintf({0x553a400, 0x2}, {0x28002a34bd08, 0x1, 0x1})\n    \t/usr/lib/golang/src/fmt/print.go:239 +0x53 fp=0x28002a34ba78 sp=0x28002a34ba20 pc=0x1fda6b3\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x280029e454e0, {0x59f0b40, 0x280029beca50}, {0x5a00300, 0x280029ae2040}, 0x28002a830c40)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:160 +0x8d8 fp=0x28002a34bd60 sp=0x28002a34ba78 pc=0x391bad8\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x2d20414320746f6f?, {0x59f0b40?, 0x280029beca50?}, {0x5a00300?, 0x280029ae2040?}, 0x43437a4b4349494d?)\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x28002a34be88 sp=0x28002a34bd60 pc=0x3929c16\n    github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n    \t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x28002a34bf78 sp=0x28002a34be88 pc=0x392984a\n    golang.org/x/sync/errgroup.(*Group).Go.func1()\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a34bfe0 sp=0x28002a34bf78 pc=0x36251d0\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a34bfe8 sp=0x28002a34bfe0 pc=0x1f40801\n    created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 137\n    \t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\n    \n    goroutine 142 gp=0x28002a84af00 m=nil [select]:\n    runtime.gopark(0x28002a333de0?, 0x2?, 0x8?, 0xb8?, 0x28002a333dbc?)\n    \t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a333c58 sp=0x28002a333c38 pc=0x1f38d2e\n    runtime.selectgo(0x28002a333de0, 0x28002a333db8, 0x280029afc008?, 0x1, 0x1f3665f?, 0x1)\n    \t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a333d88 sp=0x28002a333c58 pc=0x1f152a5\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.(*iterativeDataset).sendTable(0x280029becf00, {0x5a19be0, 0x28002ab876c0})\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:378 +0xf3 fp=0x28002a333e10 sp=0x28002a333d88 pc=0x38d8d33\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.handleDataTable(0x280029becf00, 0x28002a3b2b40)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:324 +0x2c6 fp=0x28002a333ec8 sp=0x28002a333e10 pc=0x38d8686\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x280029becf00)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:167 +0x148 fp=0x28002a333f30 sp=0x28002a333ec8 pc=0x38d7b48\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x280029becf00, 0x28002a977da0)\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x28002a333fc0 sp=0x28002a333f30 pc=0x38d788c\n    github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x28002a333fe0 sp=0x28002a333fc0 pc=0x38d751b\n    runtime.goexit({})\n    \t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a333fe8 sp=0x28002a333fe0 pc=0x1f40801\n    created by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 140\n    \t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc\n    \n    rax    0x0\n    rbx    0x7f5e554a8640\n    rcx    0x7f5e9c16994c\n    rdx    0x6\n    rdi    0x10a66\n    rsi    0x10a67\n    rbp    0x10a67\n    rsp    0x7f5e554a7740\n    r8     0x7f5e554a7810\n    r9     0x7f5e9c28f4e0\n    r10    0x8\n    r11    0x246\n    r12    0x6\n    r13    0x7f5e554a7b10\n    r14    0x280029a48b40\n    r15    0xffffffffffffffff\n    rip    0x7f5e9c16994c\n    rflags 0x246\n    cs     0x33\n    fs     0x0\n    gs     0x0\n    \n    {\n        s: \"must-gather CLI test failures:\\n  query-basic: command failed: exit status 1\\noutput: \\x1b[37m[21:33:14.493]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:23.438]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\\\\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:33:21.8572874Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\\\\\\\", ActivityId:\\\\\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\\\\\", SubActivityId:\\\\\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \\\\u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:23.438]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 50/289 failed (KQL: let _startTime = datetime(2026-10-15T01:38:14.4931102Z);\\\\nlet _endTime = datetime(2026-10-15T01:43:14.4931101Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-15T01:38:14.4931102Z);\\\\nlet _endTime = datetime(2026-10-15T01:43:14.4931101Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:23.712]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:23.712]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:24.208]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 1498,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[21:33:24.208]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:24.217]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[21:33:40.606]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n  \\\"commits\\\": 793\\n}\\x1b[0m\\n\\x1b[37m[21:33:40.607]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:40.961]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:33:41.088]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j8700672-mgmt-1\\\",\\n    \\\"ci01-j8700672-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:33:41.127]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespaces\\\": [\\n    \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:33:41.127]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[21:33:41.127]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:41.317]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:41.478]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:41.623]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:43.044]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\\\\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:33:21.8572874Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\\\\\\\", ActivityId:\\\\\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\\\\\", SubActivityId:\\\\\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \\\\u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3888395104\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:43.044]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:14.4931102Z);\\\\nlet _endTime = datetime(2026-10-15T21:33:14.4931106Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:33:21.8572874Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2\\\\\\\", ActivityId:\\\\\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\\\\\", SubActivityId:\\\\\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"26d6d205-1ba0-4e79-9fb2-b02e2df15164\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;5e59f83e-2fe6-45d6-9cea-ddfdce7b22b2 ARID=26d6d205-1ba0-4e79-9fb2-b02e2df15164 \\\\u003e GW.Http.CallContext/26d6d205-1ba0-4e79-9fb2-b02e2df15164)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n  query-skip-hcp-logs: command failed: exit status 1\\noutput: \\x1b[37m[21:33:43.407]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:49.648]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\\\\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:33:47.6716888Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\\\\\\\", ActivityId:\\\\\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\\\\\", SubActivityId:\\\\\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \\\\u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:49.648]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 47/289 failed (KQL: let _startTime = datetime(2026-10-15T01:23:43.4070857Z);\\\\nlet _endTime = datetime(2026-10-15T01:28:43.4070856Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:49.744]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:49.744]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:50.271]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 1498,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[21:33:50.271]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:33:50.281]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[21:34:08.647]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n  \\\"commits\\\": 793\\n}\\x1b[0m\\n\\x1b[37m[21:34:08.648]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:09.212]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:34:09.309]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j8700672-mgmt-1\\\",\\n    \\\"ci01-j8700672-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:34:09.470]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespaces\\\": [\\n    \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:34:09.470]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[21:34:09.470]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:09.615]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:10.183]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:10.617]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:12.156]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\\\\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:33:47.6716888Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\\\\\\\", ActivityId:\\\\\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\\\\\", SubActivityId:\\\\\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \\\\u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3807605843\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:12.157]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:33:43.4070857Z);\\\\nlet _endTime = datetime(2026-10-15T21:33:43.4070860Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:33:47.6716888Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52\\\\\\\", ActivityId:\\\\\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\\\\\", SubActivityId:\\\\\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"4f13c479-3f1c-438a-b976-532417bcd103\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;c413d1ee-239d-43f5-8bca-41f261cb5b52 ARID=4f13c479-3f1c-438a-b976-532417bcd103 \\\\u003e GW.Http.CallContext/4f13c479-3f1c-438a-b976-532417bcd103)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n  query-skip-kubernetes-events: command failed: exit status 1\\noutput: \\x1b[37m[21:34:12.559]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:18.220]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\\\\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:34:16.3210435Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\\\\\\\", ActivityId:\\\\\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\\\\\", SubActivityId:\\\\\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \\\\u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:18.221]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 47/289 failed (KQL: let _startTime = datetime(2026-10-15T01:24:12.5582990Z);\\\\nlet _endTime = datetime(2026-10-15T01:29:12.5582989Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:18.221]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:18.221]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:18.723]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 1498,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[21:34:18.724]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:18.734]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[21:34:35.437]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:35.845]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:34:36.194]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j8700672-mgmt-1\\\",\\n    \\\"ci01-j8700672-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:34:36.313]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespaces\\\": [\\n    \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:34:36.313]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[21:34:36.313]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:36.544]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:36.863]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:37.093]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:38.438]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\\\\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:34:16.3210435Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\\\\\\\", ActivityId:\\\\\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\\\\\", SubActivityId:\\\\\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \\\\u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-3673615767\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:38.438]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:12.5582990Z);\\\\nlet _endTime = datetime(2026-10-15T21:34:12.5582994Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:34:16.3210435Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be\\\\\\\", ActivityId:\\\\\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\\\\\", SubActivityId:\\\\\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"3a55dd5d-6dec-4696-91d8-21bd48f1fd94\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;3a0fc8eb-a50e-4de0-bb86-267bff2c78be ARID=3a55dd5d-6dec-4696-91d8-21bd48f1fd94 \\\\u003e GW.Http.CallContext/3a55dd5d-6dec-4696-91d8-21bd48f1fd94)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\"\\n}\\x1b[0m\\n\\n  query-collect-systemd-logs: command failed: exit status 1\\noutput: \\x1b[37m[21:34:38.772]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:46.539]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\\\\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:34:46.3046137Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\\\\\\\", ActivityId:\\\\\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\\\\\", SubActivityId:\\\\\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \\\\u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:46.539]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 45/289 failed (KQL: let _startTime = datetime(2026-10-15T01:14:38.7716298Z);\\\\nlet _endTime = datetime(2026-10-15T01:19:38.7716297Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8700672-svc', 'ci01-j8700672-mgmt-1', 'ci01-j8700672-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:47.129]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:47.129]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:47.633]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 1498,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[21:34:47.633]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:47.642]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[21:34:59.291]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:34:59.911]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:35:00.221]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j8700672-mgmt-1\\\",\\n    \\\"ci01-j8700672-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:35:00.360]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespaces\\\": [\\n    \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n    \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[21:35:00.360]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[21:35:00.360]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:35:00.702]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/klusterlet-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:35:01.111]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:35:01.638]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8700672-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfevok49t0kjg6c48lr24m7jgn4lk62-o2r4g2r6r2j5p5g\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:35:02.339]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\\\\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:34:46.3046137Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\\\\\\\", ActivityId:\\\\\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\\\\\", SubActivityId:\\\\\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \\\\u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-4003234904\\\"\\n}\\x1b[0m\\n\\x1b[37m[21:35:02.340]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T21:34:38.7716298Z);\\\\nlet _endTime = datetime(2026-10-15T21:34:38.7716302Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-cz52mf4drcbt'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T21:34:46.3046137Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d\\\\\\\", ActivityId:\\\\\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\\\\\", SubActivityId:\\\\\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"fc8708b2-6002-407d-a8da-d09e7eda9a7c\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;1ffcb862-68bb-41f2-9a5c-3d7c15e03a2d ARID=fc8708b2-6002-407d-a8da-d09e7eda9a7c \\\\u003e GW.Http.CallContext/fc8708b2-6002-407d-a8da-d09e7eda9a7c)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\"\\n}\\x1b[0m\\n\\n  query-skip-hcp-and-events: command failed: exit status 2\\noutput: \\x1b[37m[21:35:02.599]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-1950984811\\\"\\n}\\x1b[0m\\nruntime/cgo: pthread_create failed: Resource temporarily unavailable\\nSIGABRT: abort\\nPC=0x7f80df60594c m=4 sigcode=18446744073709551610\\n\\ngoroutine 0 gp=0x14197f1334a0 m=4 mp=0x14197f21a008 [idle]:\\nruntime: g 0 gp=0x14197f1334a0: unknown pc 0x7f80df60594c\\nstack: frame={sp:0x7f808bffe790, fp:0x0} stack=[0x7f808b7ff1e0,0x7f808bffede0)\\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\\n00007f808bffe690: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6a0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6b0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6c0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6d0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6e0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6f0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe700: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe710: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe720: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe730: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe740: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe750: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe760: 00000000 00000000  00007f80 df612fa9  ........./a.....\\n00007f808bffe770: 00000000 00000000  00000000 00000011  ................\\n00007f808bffe780: 00000000 00000130  00007f80 df60593e  0.......\u003eY`.....\\n00007f808bffe790: 00000000 00000120  00000000 00000000   ...............\\n                  ^ SP\\n00007f808bffe7a0: 00010000 00000013  00000000 00000000  ................\\n00007f808bffe7b0: 00000000 00000000  00007f80 df688cff  ..........h.....\\n00007f808bffe7c0: 00007f80 89ffb910  00007f80 df688cb1  ..........h.....\\n00007f808bffe7d0: 00007f80 89ffb640  00007f80 8bffeaa0  @...............\\n00007f808bffe7e0: 00007f80 8bffe90e  00007f80 8bffe90f  ................\\n00007f808bffe7f0: 00007f80 89ffb640  00007f80 df6037f5  @........7`.....\\n00007f808bffe800: 00007f80 84000020  00000000 00000120   ....... .......\\n00007f808bffe810: 00000000 003d0f00  701e1ebb c8982900  ..=......).....p\\n00007f808bffe820: 00007f80 8bfff640  00000000 00000006  @...............\\n00007f808bffe830: 00007f80 84000c90  00007f80 8bffeb60  ........`.......\\n00007f808bffe840: 00001419 7f1334a0  00007f80 df5b8646  .4......F.[.....\\n00007f808bffe850: 00007f80 df775e90  00007f80 df5a27f3  .^w......'Z.....\\n00007f808bffe860: 00000000 00000020  701e1ebb c8982900   ........).....p\\n00007f808bffe870: 00007f80 897fb000  00007f80 df677ac4  .........zg.....\\n00007f808bffe880: 00000000 00000000  00000000 0000000d  ................\\nruntime: g 0 gp=0x14197f1334a0: unknown pc 0x7f80df60594c\\nstack: frame={sp:0x7f808bffe790, fp:0x0} stack=[0x7f808b7ff1e0,0x7f808bffede0)\\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\\n00007f808bffe690: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6a0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6b0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6c0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6d0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6e0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe6f0: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe700: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe710: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe720: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe730: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe740: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe750: 00000000 00000000  00000000 00000000  ................\\n00007f808bffe760: 00000000 00000000  00007f80 df612fa9  ........./a.....\\n00007f808bffe770: 00000000 00000000  00000000 00000011  ................\\n00007f808bffe780: 00000000 00000130  00007f80 df60593e  0.......\u003eY`.....\\n00007f808bffe790: 00000000 00000120  00000000 00000000   ...............\\n                  ^ SP\\n00007f808bffe7a0: 00010000 00000013  00000000 00000000  ................\\n00007f808bffe7b0: 00000000 00000000  00007f80 df688cff  ..........h.....\\n00007f808bffe7c0: 00007f80 89ffb910  00007f80 df688cb1  ..........h.....\\n00007f808bffe7d0: 00007f80 89ffb640  00007f80 8bffeaa0  @...............\\n00007f808bffe7e0: 00007f80 8bffe90e  00007f80 8bffe90f  ................\\n00007f808bffe7f0: 00007f80 89ffb640  00007f80 df6037f5  @........7`.....\\n00007f808bffe800: 00007f80 84000020  00000000 00000120   ....... .......\\n00007f808bffe810: 00000000 003d0f00  701e1ebb c8982900  ..=......).....p\\n00007f808bffe820: 00007f80 8bfff640  00000000 00000006  @...............\\n00007f808bffe830: 00007f80 84000c90  00007f80 8bffeb60  ........`.......\\n00007f808bffe840: 00001419 7f1334a0  00007f80 df5b8646  .4......F.[.....\\n00007f808bffe850: 00007f80 df775e90  00007f80 df5a27f3  .^w......'Z.....\\n00007f808bffe860: 00000000 00000020  701e1ebb c8982900   ........).....p\\n00007f808bffe870: 00007f80 897fb000  00007f80 df677ac4  .........zg.....\\n00007f808bffe880: 00000000 00000000  00000000 0000000d  ................\\n\\ngoroutine 1 gp=0x14197f1321e0 m=nil [sync.WaitGroup.Wait]:\\nruntime.gopark(0x14197fe86c00?, 0x2022a4d?, 0x10?, 0x63?, 0x14197fe86c30?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197ff32be8 sp=0x14197ff32bc8 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x14197fc98610, 0x0, 0x1, 0x0, 0x19)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197ff32c50 sp=0x14197ff32be8 pc=0x1f162b2\\nsync.runtime_SemacquireWaitGroup(0x14197fd289c0?, 0xe0?)\\n\\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x14197ff32c88 sp=0x14197ff32c50 pc=0x1f3a66e\\nsync.(*WaitGroup).Wait(0x14197fc98608)\\n\\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x14197ff32cb0 sp=0x14197ff32c88 pc=0x1f4f445\\ngolang.org/x/sync/errgroup.(*Group).Wait(0x14197fc98600)\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x14197ff32cd0 sp=0x14197ff32cb0 pc=0x362507e\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x14197f521dc0, {0x59f0b08, 0x14197f530fc0}, {0x14197fcca000, 0x15, 0x1c})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x14197ff32d90 sp=0x14197ff32cd0 pc=0x3927e05\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x14197f521dc0, {0x59f0b08, 0x14197f530fc0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:500 +0x4b08 fp=0x14197ff33968 sp=0x14197ff32d90 pc=0x39261c8\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedQueryOptions).RunQuery(0x14197f1401e0, {0x59f0b08, 0x14197f530fc0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_cmd.go:75 +0x565 fp=0x14197ff33b30 sp=0x14197ff33968 pc=0x3937005\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawQueryOptions).Run(0x0?, {0x59f0b08, 0x14197f530fc0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_options.go:178 +0x54 fp=0x14197ff33b58 sp=0x14197ff33b30 pc=0x3939d34\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryCommand.func1(0x14197f4ba008?, {0x14197f5f82a0?, 0x4?, 0x553b0e2?})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_cmd.go:45 +0x28 fp=0x14197ff33b80 sp=0x14197ff33b58 pc=0x3936a68\\ngithub.com/spf13/cobra.(*Command).execute(0x14197f4ba008, {0x14197f2c0030, 0xe, 0xf})\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x14197ff33d28 sp=0x14197ff33b80 pc=0x212e614\\ngithub.com/spf13/cobra.(*Command).ExecuteC(0x14197f625b08)\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x14197ff33e18 sp=0x14197ff33d28 pc=0x212efa5\\ngithub.com/spf13/cobra.(*Command).Execute(...)\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071\\ngithub.com/spf13/cobra.(*Command).ExecuteContext(...)\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064\\nmain.main()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x14197ff33f48 sp=0x14197ff33e18 pc=0x47ddd6a\\nruntime.main()\\n\\t/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x14197ff33fe0 sp=0x14197ff33f48 pc=0x1f01975\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197ff33fe8 sp=0x14197ff33fe0 pc=0x1f40801\\n\\ngoroutine 2 gp=0x14197f132d20 m=nil [force gc (idle)]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f2fa8 sp=0x14197f1f2f88 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.forcegchelper()\\n\\t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x14197f1f2fe0 sp=0x14197f1f2fa8 pc=0x1f01c93\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f2fe8 sp=0x14197f1f2fe0 pc=0x1f40801\\ncreated by runtime.init.7 in goroutine 1\\n\\t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\\n\\ngoroutine 18 gp=0x14197f2843c0 m=nil [GC sweep wait]:\\nruntime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1ee788 sp=0x14197f1ee768 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.bgsweep(0x14197f292000)\\n\\t/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x14197f1ee7c8 sp=0x14197f1ee788 pc=0x1ee9f11\\nruntime.gcenable.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x14197f1ee7e0 sp=0x14197f1ee7c8 pc=0x1edadd7\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1ee7e8 sp=0x14197f1ee7e0 pc=0x1f40801\\ncreated by runtime.gcenable in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\\n\\ngoroutine 19 gp=0x14197f2845a0 m=nil [GC scavenge wait]:\\nruntime.gopark(0x138d6d?, 0x3979fd?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1eef78 sp=0x14197f1eef58 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.(*scavengerState).park(0x82979c0)\\n\\t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x14197f1eefa8 sp=0x14197f1eef78 pc=0x1ee77e9\\nruntime.bgscavenge(0x14197f292000)\\n\\t/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x14197f1eefc8 sp=0x14197f1eefa8 pc=0x1ee7d99\\nruntime.gcenable.gowrap2()\\n\\t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x14197f1eefe0 sp=0x14197f1eefc8 pc=0x1edad97\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1eefe8 sp=0x14197f1eefe0 pc=0x1f40801\\ncreated by runtime.gcenable in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\\n\\ngoroutine 20 gp=0x14197f284780 m=nil [GOMAXPROCS updater (idle)]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1ef788 sp=0x14197f1ef768 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.updateMaxProcsGoroutine()\\n\\t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x14197f1ef7e0 sp=0x14197f1ef788 pc=0x1f10267\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1ef7e8 sp=0x14197f1ef7e0 pc=0x1f40801\\ncreated by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\\n\\t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\\n\\ngoroutine 21 gp=0x14197f284d20 m=nil [finalizer wait]:\\nruntime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x14197f1f2601?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f2620 sp=0x14197f1f2600 pc=0x1f38d2e\\nruntime.runFinalizers()\\n\\t/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x14197f1f27e0 sp=0x14197f1f2620 pc=0x1ed9cc7\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f27e8 sp=0x14197f1f27e0 pc=0x1f40801\\ncreated by runtime.createfing in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d\\n\\ngoroutine 22 gp=0x14197f47c960 m=nil [cleanup wait]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1eff68 sp=0x14197f1eff48 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.(*cleanupQueue).dequeue(0x82980e0)\\n\\t/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x14197f1effa0 sp=0x14197f1eff68 pc=0x1ed6074\\nruntime.runCleanups()\\n\\t/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x14197f1effe0 sp=0x14197f1effa0 pc=0x1ed66e5\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1effe8 sp=0x14197f1effe0 pc=0x1f40801\\ncreated by runtime.(*cleanupQueue).createGs in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5\\n\\ngoroutine 23 gp=0x14197f5a30e0 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125b862b4ec7?, 0x1?, 0x21?, 0xab?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f0740 sp=0x14197f1f0720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x14197f29d9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f07c8 sp=0x14197f1f0740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f07e0 sp=0x14197f1f07c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f07e8 sp=0x14197f1f07e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 3 gp=0x14197f133680 m=nil [runnable]:\\nruntime.gopark(0x125b8625d223?, 0x3?, 0x89?, 0x19?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f3740 sp=0x14197f1f3720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x14197f29d9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f37c8 sp=0x14197f1f3740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f37e0 sp=0x14197f1f37c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f37e8 sp=0x14197f1f37e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 34 gp=0x14197f604000 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125b862e8e02?, 0x3?, 0xe6?, 0x2d?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60a740 sp=0x14197f60a720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x14197f29d9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f60a7c8 sp=0x14197f60a740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f60a7e0 sp=0x14197f60a7c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60a7e8 sp=0x14197f60a7e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 24 gp=0x14197f5a32c0 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125b862118da?, 0x1?, 0x4d?, 0x9a?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f0f40 sp=0x14197f1f0f20 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x14197f29d9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f0fc8 sp=0x14197f1f0f40 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f0fe0 sp=0x14197f1f0fc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f0fe8 sp=0x14197f1f0fe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 4 gp=0x14197f133860 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125b7fe90140?, 0x3?, 0x20?, 0x3a?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f3f40 sp=0x14197f1f3f20 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x14197f29d9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f3fc8 sp=0x14197f1f3f40 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f3fe0 sp=0x14197f1f3fc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f3fe8 sp=0x14197f1f3fe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 35 gp=0x14197f6041e0 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125b862a34bd?, 0x3?, 0x4?, 0x15?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60af40 sp=0x14197f60af20 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x14197f29d9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f60afc8 sp=0x14197f60af40 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f60afe0 sp=0x14197f60afc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60afe8 sp=0x14197f60afe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 25 gp=0x14197f5a34a0 m=3 mp=0x14197f1a5808 [GC mark termination]:\\nruntime.systemstack_switch()\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x14197f1f16a0 sp=0x14197f1f1690 pc=0x1f3ebc8\\nruntime.forEachP(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:2112\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x14197f1f1740 sp=0x14197f1f16a0 pc=0x1edbc34\\nruntime.gcBgMarkWorker(0x14197f29d9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x14197f1f17c8 sp=0x14197f1f1740 pc=0x1edda88\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f17e0 sp=0x14197f1f17c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f17e8 sp=0x14197f1f17e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 5 gp=0x14197f133a40 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125b8624ea9b?, 0x1?, 0xfd?, 0x7a?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1f4740 sp=0x14197f1f4720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x14197f29d9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x14197f1f47c8 sp=0x14197f1f4740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x14197f1f47e0 sp=0x14197f1f47c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1f47e8 sp=0x14197f1f47e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 50 gp=0x14197f768f00 m=nil [select, locked to thread]:\\nruntime.gopark(0x14197f60c7a8?, 0x2?, 0x1?, 0x8f?, 0x14197f60c794?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f60c628 sp=0x14197f60c608 pc=0x1f38d2e\\nruntime.selectgo(0x14197f60c7a8, 0x14197f60c790, 0x0?, 0x0, 0x0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f60c758 sp=0x14197f60c628 pc=0x1f152a5\\nruntime.ensureSigM.func1()\\n\\t/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x14197f60c7e0 sp=0x14197f60c758 pc=0x1f328e8\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f60c7e8 sp=0x14197f60c7e0 pc=0x1f40801\\ncreated by runtime.ensureSigM in goroutine 1\\n\\t/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5\\n\\ngoroutine 26 gp=0x14197f3b34a0 m=7 mp=0x14197f55b008 [syscall]:\\nruntime.notetsleepg(0x82bdea0, 0xffffffffffffffff)\\n\\t/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x14197f6077a0 sp=0x14197f607778 pc=0x1ece509\\nos/signal.signal_recv()\\n\\t/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x14197f6077c0 sp=0x14197f6077a0 pc=0x1f3b298\\nos/signal.loop()\\n\\t/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x14197f6077e0 sp=0x14197f6077c0 pc=0x2030553\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f6077e8 sp=0x14197f6077e0 pc=0x1f40801\\ncreated by os/signal.Notify.func1.1 in goroutine 1\\n\\t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\\n\\ngoroutine 27 gp=0x14197f6894a0 m=nil [select]:\\nruntime.gopark(0x14197f607f98?, 0x2?, 0x68?, 0x81?, 0x14197f607f8c?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f607e28 sp=0x14197f607e08 pc=0x1f38d2e\\nruntime.selectgo(0x14197f607f98, 0x14197f607f88, 0x0?, 0x0, 0x0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f607f58 sp=0x14197f607e28 pc=0x1f152a5\\nos/signal.NotifyContext.func1()\\n\\t/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x14197f607fe0 sp=0x14197f607f58 pc=0x202ff9c\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f607fe8 sp=0x14197f607fe0 pc=0x1f40801\\ncreated by os/signal.NotifyContext in goroutine 1\\n\\t/usr/lib/golang/src/os/signal/signal.go:291 +0x158\\n\\ngoroutine 224 gp=0x14197f689680 m=nil [semacquire]:\\nruntime.gopark(0x7f80904adba8?, 0x1eed7e0?, 0xc0?, 0x68?, 0x4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419800b5b58 sp=0x1419800b5b38 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x1419800b5bc0 sp=0x1419800b5b58 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x1419800b5c60 sp=0x1419800b5bc0 pc=0x1edbb12\\nruntime.gcAssistAlloc(0x14197f689680)\\n\\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x1419800b5cf0 sp=0x1419800b5c60 pc=0x1ee06b6\\nruntime.deductAssistCredit(0x7ffce51f9080?)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x1419800b5d10 sp=0x1419800b5cf0 pc=0x1ed10a6\\nruntime.mallocgc(0x7fe00, 0x4af32a0, 0x1)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x1419800b5d40 sp=0x1419800b5d10 pc=0x1f35e07\\nruntime.makeslice(0x14197fc9d0b0?, 0x141983a1fe00?, 0x20000?)\\n\\t/usr/lib/golang/src/runtime/slice.go:117 +0x49 fp=0x1419800b5d68 sp=0x1419800b5d40 pc=0x1f3b489\\nencoding/json.(*Decoder).refill(0x14197f658c80)\\n\\t/usr/lib/golang/src/encoding/json/stream.go:161 +0xee fp=0x1419800b5db8 sp=0x1419800b5d68 pc=0x200d48e\\nencoding/json.(*Decoder).readValue(0x14197f658c80)\\n\\t/usr/lib/golang/src/encoding/json/stream.go:142 +0x85 fp=0x1419800b5e08 sp=0x1419800b5db8 pc=0x200d125\\nencoding/json.(*Decoder).Decode(0x14197f658c80, {0x4cfb340, 0x14197ffa44c0})\\n\\t/usr/lib/golang/src/encoding/json/stream.go:65 +0x75 fp=0x1419800b5e38 sp=0x1419800b5e08 pc=0x200cdf5\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readPrimaryTable(0x14197f8c54f0, 0x14197fb50000)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:264 +0x16e fp=0x1419800b5ec8 sp=0x1419800b5e38 pc=0x38d822e\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x14197f8c54f0)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:174 +0x189 fp=0x1419800b5f30 sp=0x1419800b5ec8 pc=0x38d7b89\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x14197f8c54f0, 0x14198007be90)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x1419800b5fc0 sp=0x1419800b5f30 pc=0x38d788c\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x1419800b5fe0 sp=0x1419800b5fc0 pc=0x38d751b\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1419800b5fe8 sp=0x1419800b5fe0 pc=0x1f40801\\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 183\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc\\n\\ngoroutine 87 gp=0x14197fd3c000 m=nil [sync.WaitGroup.Wait]:\\nruntime.gopark(0x14197fcf9e00?, 0x1ed0734?, 0x20?, 0x94?, 0x7f80917bdf06?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf9da0 sp=0x14197fcf9d80 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x14197f69cc50, 0x0, 0x1, 0x0, 0x19)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fcf9e08 sp=0x14197fcf9da0 pc=0x1f162b2\\nsync.runtime_SemacquireWaitGroup(0x14197f86b2d8?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x14197fcf9e40 sp=0x14197fcf9e08 pc=0x1f3a66e\\nsync.(*WaitGroup).Wait(0x14197f69cc48)\\n\\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x14197fcf9e68 sp=0x14197fcf9e40 pc=0x1f4f445\\ngolang.org/x/sync/errgroup.(*Group).Wait(0x14197f69cc40)\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x14197fcf9e88 sp=0x14197fcf9e68 pc=0x362507e\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x14197f521d80, {0x59f0b40, 0x14197f8c4b40}, {0x14197fcca000, 0x15, 0x6f63746669687372?}, 0x14197f1c02a0)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x14197fcf9f00 sp=0x14197fcf9e88 pc=0x392976a\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x14197fcf9f78 sp=0x14197fcf9f00 pc=0x392808e\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197fcf9fe0 sp=0x14197fcf9f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf9fe8 sp=0x14197fcf9fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 77 gp=0x14197fd3c1e0 m=nil [IO wait]:\\nruntime.gopark(0x14197fb3e4a0?, 0x1edb176?, 0x80?, 0xed?, 0xb?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f202808 sp=0x14197f2027e8 pc=0x1f38d2e\\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x14197f202840 sp=0x14197f202808 pc=0x1ef9e57\\ninternal/poll.runtime_pollWait(0x7f8098109000, 0x72)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x14197f202860 sp=0x14197f202840 pc=0x1f375c5\\ninternal/poll.(*pollDesc).wait(0x14197f323a80?, 0x14197f34d300?, 0x0)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x14197f202888 sp=0x14197f202860 pc=0x1fbd167\\ninternal/poll.(*pollDesc).waitRead(...)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\\ninternal/poll.(*FD).Read(0x14197f323a80, {0x14197f34d300, 0x1300, 0x1300})\\n\\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x14197f202920 sp=0x14197f202888 pc=0x1fbe4ce\\nnet.(*netFD).Read(0x14197f323a80, {0x14197f34d300?, 0x14197f34d30a?, 0x5?})\\n\\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x14197f202968 sp=0x14197f202920 pc=0x20798c5\\nnet.(*conn).Read(0x14197f1300d8, {0x14197f34d300?, 0x7f809057a868?, 0x7f80df53a6d8?})\\n\\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x14197f2029b0 sp=0x14197f202968 pc=0x208b7a5\\ncrypto/tls.(*atLeastReader).Read(0x14197f821a10, {0x14197f34d300?, 0x14197f202a98?, 0x1efcc28?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x14197f2029f8 sp=0x14197f2029b0 pc=0x2222ddb\\nbytes.(*Buffer).ReadFrom(0x14197f8a57a8, {0x59a3680, 0x14197f821a10})\\n\\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x14197f202a50 sp=0x14197f2029f8 pc=0x1fe9258\\ncrypto/tls.(*Conn).readFromUntil(0x14197f8a5508, {0x59a32c0, 0x14197f1300d8}, 0x222764c?)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x14197f202a88 sp=0x14197f202a50 pc=0x2222fbe\\ncrypto/tls.(*Conn).readRecordOrCCS(0x14197f8a5508, 0x0)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x14197f202c68 sp=0x14197f202a88 pc=0x222009b\\ncrypto/tls.(*Conn).readRecord(...)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:588\\ncrypto/tls.(*Conn).Read(0x14197f8a5508, {0x14197fa26000, 0x1000, 0x4e55540?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x14197f202cd0 sp=0x14197f202c68 pc=0x22267c5\\nnet/http.(*persistConn).Read(0x14197f8032c0, {0x14197fa26000?, 0x59a0ea0?, 0x81260e0?})\\n\\t/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x14197f202d30 sp=0x14197f202cd0 pc=0x2319d07\\nbufio.(*Reader).fill(0x14197f29b620)\\n\\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f202d68 sp=0x14197f202d30 pc=0x20a6c63\\nbufio.(*Reader).Peek(0x14197f29b620, 0x1)\\n\\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x14197f202d80 sp=0x14197f202d68 pc=0x20a6df3\\nnet/http.(*persistConn).readLoop(0x14197f8032c0)\\n\\t/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x14197f202fc8 sp=0x14197f202d80 pc=0x231a812\\nnet/http.(*Transport).dialConn.gowrap2()\\n\\t/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x14197f202fe0 sp=0x14197f202fc8 pc=0x2319197\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f202fe8 sp=0x14197f202fe0 pc=0x1f40801\\ncreated by net/http.(*Transport).dialConn in goroutine 9\\n\\t/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac\\n\\ngoroutine 7 gp=0x14197fe025a0 m=nil [sync.Mutex.Lock]:\\nruntime.gopark(0x14197f9d3c68?, 0x14197f9d3c70?, 0x94?, 0x69?, 0x14197f9d3c78?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f9d3c00 sp=0x14197f9d3be0 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x14197fccaad4, 0x0, 0x3, 0x2, 0x16)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197f9d3c68 sp=0x14197f9d3c00 pc=0x1f162b2\\ninternal/sync.runtime_SemacquireMutex(0x1000?, 0xd0?, 0x1ed11a5?)\\n\\t/usr/lib/golang/src/runtime/sema.go:95 +0x25 fp=0x14197f9d3ca0 sp=0x14197f9d3c68 pc=0x1f3a545\\ninternal/sync.(*Mutex).lockSlow(0x14197fccaad0)\\n\\t/usr/lib/golang/src/internal/sync/mutex.go:149 +0x15d fp=0x14197f9d3cf0 sp=0x14197f9d3ca0 pc=0x1f4d31d\\ninternal/sync.(*Mutex).Lock(...)\\n\\t/usr/lib/golang/src/internal/sync/mutex.go:70\\nsync.(*Mutex).Lock(...)\\n\\t/usr/lib/golang/src/sync/mutex.go:46\\nnet/http.(*http2clientConnReadLoop).streamByID(0x22babb5?, 0x1b, 0x1)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:10083 +0x5b fp=0x14197f9d3d48 sp=0x14197f9d3cf0 pc=0x22e10db\\nnet/http.(*http2clientConnReadLoop).processData(0x14197f9d3fa8, 0x14197fc9d200)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9939 +0x45 fp=0x14197f9d3e18 sp=0x14197f9d3d48 pc=0x22e05e5\\nnet/http.(*http2clientConnReadLoop).run(0x14197f9d3fa8)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9584 +0x4b6 fp=0x14197f9d3f60 sp=0x14197f9d3e18 pc=0x22de536\\nnet/http.(*http2ClientConn).readLoop(0x14197fccaa80)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x14197f9d3fc8 sp=0x14197f9d3f60 pc=0x22dd532\\nnet/http.(*http2Transport).newClientConn.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x14197f9d3fe0 sp=0x14197f9d3fc8 pc=0x22d7057\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f9d3fe8 sp=0x14197f9d3fe0 pc=0x1f40801\\ncreated by net/http.(*http2Transport).newClientConn in goroutine 6\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5\\n\\ngoroutine 78 gp=0x14197fe02960 m=nil [select]:\\nruntime.gopark(0x14197f9f7f48?, 0x2?, 0x0?, 0xb6?, 0x14197f9f7ef4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf2d88 sp=0x14197fcf2d68 pc=0x1f38d2e\\nruntime.selectgo(0x14197fcf2f48, 0x14197f9f7ef0, 0x14197f1cd0c0?, 0x0, 0x14197f89c2a0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197fcf2eb8 sp=0x14197fcf2d88 pc=0x1f152a5\\nnet/http.(*persistConn).writeLoop(0x14197f8032c0)\\n\\t/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x14197fcf2fc8 sp=0x14197fcf2eb8 pc=0x231c426\\nnet/http.(*Transport).dialConn.gowrap3()\\n\\t/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x14197fcf2fe0 sp=0x14197fcf2fc8 pc=0x2319157\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf2fe8 sp=0x14197fcf2fe0 pc=0x1f40801\\ncreated by net/http.(*Transport).dialConn in goroutine 9\\n\\t/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe\\n\\ngoroutine 163 gp=0x14197f7694a0 m=nil [select]:\\nruntime.gopark(0x14197fdd3f08?, 0x6?, 0xa0?, 0x94?, 0x14197fdd3da4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f205c08 sp=0x14197f205be8 pc=0x1f38d2e\\nruntime.selectgo(0x14197f205f08, 0x14197fdd3d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f205d38 sp=0x14197f205c08 pc=0x1f152a5\\nnet/http.(*http2clientStream).writeRequest(0x14197f37f380, 0x14197f49e140, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x14197f205f90 sp=0x14197f205d38 pc=0x22da6a8\\nnet/http.(*http2clientStream).doRequest(0x14197f37f380, 0x14197f546100?, 0x14197f609798?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x14197f205fb8 sp=0x14197f205f90 pc=0x22d99d8\\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197f205fe0 sp=0x14197f205fb8 pc=0x22d999f\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f205fe8 sp=0x14197f205fe0 pc=0x1f40801\\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 184\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\\n\\ngoroutine 88 gp=0x14197f769a40 m=nil [semacquire]:\\nruntime.gopark(0x141980119958?, 0x1ed3e05?, 0xe8?, 0xd2?, 0x7f80904ad2e8?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x141980119908 sp=0x1419801198e8 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x141980119970 sp=0x141980119908 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x141980119a10 sp=0x141980119970 pc=0x1edbb12\\nruntime.gcAssistAlloc(0x14197f769a40)\\n\\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x141980119aa0 sp=0x141980119a10 pc=0x1ee06b6\\nruntime.deductAssistCredit(0x141983b00000?)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x141980119ac0 sp=0x141980119aa0 pc=0x1ed10a6\\nruntime.mallocgc(0x7edd, 0x0, 0x0)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x141980119af0 sp=0x141980119ac0 pc=0x1f35e07\\nruntime.slicebytetostring(0x4e537a0?, 0x141983b00000, 0x7edd)\\n\\t/usr/lib/golang/src/runtime/string.go:170 +0x77 fp=0x141980119b20 sp=0x141980119af0 pc=0x1f1e397\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x14197f8c4b40}, 0x14197f1c03f0, 0x14197f531740)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:322 +0x91b fp=0x141980119f48 sp=0x141980119b20 pc=0x3920bfb\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x141980119f78 sp=0x141980119f48 pc=0x392800c\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x141980119fe0 sp=0x141980119f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x141980119fe8 sp=0x141980119fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 110 gp=0x14197fd3cb40 m=nil [sync.Mutex.Lock]:\\nruntime.gopark(0x22b8a73?, 0x14197f7e1000?, 0xb0?, 0x4c?, 0x8160a40?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f1c7510 sp=0x14197f1c74f0 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x14197fccaad4, 0x0, 0x3, 0x2, 0x16)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197f1c7578 sp=0x14197f1c7510 pc=0x1f162b2\\ninternal/sync.runtime_SemacquireMutex(0x14197f1c75d0?, 0x66?, 0x61f?)\\n\\t/usr/lib/golang/src/runtime/sema.go:95 +0x25 fp=0x14197f1c75b0 sp=0x14197f1c7578 pc=0x1f3a545\\ninternal/sync.(*Mutex).lockSlow(0x14197fccaad0)\\n\\t/usr/lib/golang/src/internal/sync/mutex.go:149 +0x15d fp=0x14197f1c7600 sp=0x14197f1c75b0 pc=0x1f4d31d\\ninternal/sync.(*Mutex).Lock(...)\\n\\t/usr/lib/golang/src/internal/sync/mutex.go:70\\nsync.(*Mutex).Lock(...)\\n\\t/usr/lib/golang/src/sync/mutex.go:46\\nnet/http.http2transportResponseBody.Read({0x1f3ec12?}, {0x14198313d000?, 0x1eed797?, 0x14197f1c76b0?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9875 +0x209 fp=0x14197f1c7688 sp=0x14197f1c7600 pc=0x22dff89\\nbufio.(*Reader).fill(0x14197f3a00c0)\\n\\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f1c76c0 sp=0x14197f1c7688 pc=0x20a6c63\\nbufio.(*Reader).ReadByte(0x14197f3a00c0)\\n\\t/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x14197f1c76d8 sp=0x14197f1c76c0 pc=0x20a73c7\\ncompress/flate.(*decompressor).moreBits(0x141980059908)\\n\\t/usr/lib/golang/src/compress/flate/inflate.go:697 +0x24 fp=0x14197f1c7710 sp=0x14197f1c76d8 pc=0x2288824\\ncompress/flate.(*decompressor).nextBlock(0x141980059908)\\n\\t/usr/lib/golang/src/compress/flate/inflate.go:304 +0x28 fp=0x14197f1c7728 sp=0x14197f1c7710 pc=0x2286ba8\\ncompress/flate.(*decompressor).Read(0x141980059908, {0x14198314e000, 0x1000, 0x7f8090ef0f00?})\\n\\t/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x14197f1c7758 sp=0x14197f1c7728 pc=0x2286dbb\\ncompress/gzip.(*Reader).Read(0x14197f672dc8, {0x14198314e000, 0x1000, 0x1000})\\n\\t/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x14197f1c77e8 sp=0x14197f1c7758 pc=0x228c838\\ngithub.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x410d0fa76c5d7d40?, {0x14198314e000?, 0x7f8090ef0d20?, 0x52ffb40?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x14197f1c7818 sp=0x14197f1c77e8 pc=0x38b33e2\\nbufio.(*Reader).fill(0x14197f3a0120)\\n\\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14197f1c7850 sp=0x14197f1c7818 pc=0x20a6c63\\nbufio.(*Reader).Peek(0x14197f3a0120, 0x1)\\n\\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x14197f1c7868 sp=0x14197f1c7850 pc=0x20a6df3\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.validateJsonResponse(0x14197f3a0120)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:38 +0x25 fp=0x14197f1c78b8 sp=0x14197f1c7868 pc=0x38d6be5\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.newFrameReader({0x59db110, 0x14197fe1e040}, {0x59f0b40, 0x14197f748140})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:26 +0x188 fp=0x14197f1c7960 sp=0x14197f1c78b8 pc=0x38d6a08\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset({0x59f0bb0?, 0x14197f4797a0?}, {0x59db110, 0x14197fe1e040}, 0x1, 0x3e8, 0x1)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:66 +0x19e fp=0x14197f1c79e0 sp=0x14197f1c7960 pc=0x38d737e\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x14197f521d40, {0x59f0bb0, 0x14197f4797a0}, {0x14197f49a330, 0xb}, 0x14197fe1e700, {0x0?, 0x2?, 0x0?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:167 +0x2b7 fp=0x14197f1c7a78 sp=0x14197f1c79e0 pc=0x38e4f17\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197f7a2600}, 0x14197f1c02a0)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x14197f1c7d60 sp=0x14197f1c7a78 pc=0x391b5c5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x14197f521d80, {0x59f0b40, 0x14197fdd8140}, {0x5a00340, 0x14197fc1e5d0}, 0x14197f1c02a0)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:98 +0x366 fp=0x14197f1c7e88 sp=0x14197f1c7d60 pc=0x3929ec6\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14197f1c7f78 sp=0x14197f1c7e88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197f1c7fe0 sp=0x14197f1c7f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f1c7fe8 sp=0x14197f1c7fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 89 gp=0x14197fd82960 m=nil [select]:\\nruntime.gopark(0x1419800b7d00?, 0x2?, 0xd7?, 0xfd?, 0x1419800b7c74?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419800b7ac8 sp=0x1419800b7aa8 pc=0x1f38d2e\\nruntime.selectgo(0x1419800b7d00, 0x1419800b7c70, 0x1419801521c0?, 0x0, 0x14197f5daba0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x1419800b7bf8 sp=0x1419800b7ac8 pc=0x1f152a5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x14197f521d80?, {0x59f0b40, 0x14197f8c4b40}, 0x14197f1c02a0, 0x14197f1c03f0)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x1419800b7f10 sp=0x1419800b7bf8 pc=0x392836e\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x1419800b7f78 sp=0x1419800b7f10 pc=0x3927f3d\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x1419800b7fe0 sp=0x1419800b7f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1419800b7fe8 sp=0x1419800b7fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 175 gp=0x14197fd3d2c0 m=nil [sync.Cond.Wait]:\\nruntime.gopark(0x22c2d5b?, 0x14197fe1c080?, 0x0?, 0xe0?, 0x14197f777000?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14198012b978 sp=0x14198012b958 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nsync.runtime_notifyListWait(0x14197f37f3c8, 0x3c)\\n\\t/usr/lib/golang/src/runtime/sema.go:617 +0x1b3 fp=0x14198012b9c8 sp=0x14198012b978 pc=0x1f3a8f3\\nsync.(*Cond).Wait(0x1000?)\\n\\t/usr/lib/golang/src/sync/cond.go:71 +0x73 fp=0x14198012ba00 sp=0x14198012b9c8 pc=0x1f4d673\\nnet/http.(*http2pipe).Read(0x14197f37f3b0, {0x14197ff7e000, 0x1000, 0x1000})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:3928 +0xdb fp=0x14198012ba68 sp=0x14198012ba00 pc=0x22c2c1b\\nnet/http.http2transportResponseBody.Read({0x4a54?}, {0x14197ff7e000?, 0x14198012bb80?, 0x2287949?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9852 +0x59 fp=0x14198012baf0 sp=0x14198012ba68 pc=0x22dfdd9\\nbufio.(*Reader).fill(0x14197fcaa000)\\n\\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14198012bb28 sp=0x14198012baf0 pc=0x20a6c63\\nbufio.(*Reader).ReadByte(0x14197fcaa000)\\n\\t/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x14198012bb40 sp=0x14198012bb28 pc=0x20a73c7\\ncompress/flate.(*decompressor).moreBits(0x141980056008)\\n\\t/usr/lib/golang/src/compress/flate/inflate.go:697 +0x24 fp=0x14198012bb78 sp=0x14198012bb40 pc=0x2288824\\ncompress/flate.(*decompressor).nextBlock(0x141980056008)\\n\\t/usr/lib/golang/src/compress/flate/inflate.go:304 +0x28 fp=0x14198012bb90 sp=0x14198012bb78 pc=0x2286ba8\\ncompress/flate.(*decompressor).Read(0x141980056008, {0x141980034b13, 0x4ed, 0xb13?})\\n\\t/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x14198012bbc0 sp=0x14198012bb90 pc=0x2286dbb\\ncompress/gzip.(*Reader).Read(0x141980000b08, {0x141980034b13, 0x4ed, 0x4ed})\\n\\t/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x14198012bc50 sp=0x14198012bbc0 pc=0x228c838\\ngithub.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x14198012bcf0?, {0x141980034b13?, 0x1000?, 0x0?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x14198012bc80 sp=0x14198012bc50 pc=0x38b33e2\\nbufio.(*Reader).fill(0x141980807010)\\n\\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x14198012bcb8 sp=0x14198012bc80 pc=0x20a6c63\\nbufio.(*Reader).ReadSlice(0x141980807010, 0xa)\\n\\t/usr/lib/golang/src/bufio/bufio.go:377 +0x29 fp=0x14198012bd00 sp=0x14198012bcb8 pc=0x20a7769\\nbufio.(*Reader).collectFragments(0x141980807010, 0xa)\\n\\t/usr/lib/golang/src/bufio/bufio.go:452 +0x85 fp=0x14198012bde0 sp=0x14198012bd00 pc=0x20a7b85\\nbufio.(*Reader).ReadBytes(0x4bbc5a0?, 0xa0?)\\n\\t/usr/lib/golang/src/bufio/bufio.go:479 +0x17 fp=0x14198012be60 sp=0x14198012bde0 pc=0x20a7e77\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.(*frameReader).advance(0x141980807000)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:64 +0x45 fp=0x14198012beb0 sp=0x14198012be60 pc=0x38d6d45\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readRoutine(0x141980807000, 0x14197f8c8370)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:85 +0x51 fp=0x14198012bfc0 sp=0x14198012beb0 pc=0x38d7591\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap2()\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x1b fp=0x14198012bfe0 sp=0x14198012bfc0 pc=0x38d74db\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14198012bfe8 sp=0x14198012bfe0 pc=0x1f40801\\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 184\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x24a\\n\\ngoroutine 183 gp=0x14197fd3d860 m=nil [semacquire]:\\nruntime.gopark(0x7f8090f36c08?, 0x1eed7e0?, 0x80?, 0x43?, 0x2?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fac5208 sp=0x14197fac51e8 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fac5270 sp=0x14197fac5208 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x14197fac5310 sp=0x14197fac5270 pc=0x1edbb12\\nruntime.gcAssistAlloc(0x14197fd3d860)\\n\\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x14197fac53a0 sp=0x14197fac5310 pc=0x1ee06b6\\nruntime.deductAssistCredit(0x14197fac53f0?)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x14197fac53c0 sp=0x14197fac53a0 pc=0x1ed10a6\\nruntime.mallocgc(0x5500, 0x0, 0x0)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x14197fac53f0 sp=0x14197fac53c0 pc=0x1f35e07\\nruntime.growslice(0x141983adc000, 0x14197fac54a8?, 0x1ed3be5?, 0x82bed80?, 0x1ed0417?)\\n\\t/usr/lib/golang/src/runtime/slice.go:265 +0x573 fp=0x14197fac5460 sp=0x14197fac53f0 pc=0x1f3ba93\\nstrings.(*Builder).Write(0x14197f569ea0, {0x1419839fd000, 0x1000, 0x1000000010765?})\\n\\t/usr/lib/golang/src/strings/builder.go:89 +0x90 fp=0x14197fac54b0 sp=0x14197fac5460 pc=0x1ff0cb0\\nbufio.(*Writer).Flush(0x14197fac5698)\\n\\t/usr/lib/golang/src/bufio/bufio.go:642 +0x52 fp=0x14197fac54f8 sp=0x14197fac54b0 pc=0x20a8832\\nbufio.(*Writer).WriteString(0x14197fac5698, {0x553a4f6?, 0x553a9e6?})\\n\\t/usr/lib/golang/src/bufio/bufio.go:766 +0x165 fp=0x14197fac5568 sp=0x14197fac54f8 pc=0x20a8f65\\nencoding/csv.(*Writer).Write(0x14197fac5688, {0x14198384d480?, 0x4, 0x141983909500?})\\n\\t/usr/lib/golang/src/encoding/csv/writer.go:92 +0x2b4 fp=0x14197fac55d0 sp=0x14197fac5568 pc=0x20ad3d4\\ngithub.com/Azure/azure-kusto-go/azkustodata/query.(*row).String(0x5e?)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/row_impl.go:101 +0x21f fp=0x14197fac56e8 sp=0x14197fac55d0 pc=0x38cf67f\\nfmt.(*pp).handleMethods(0x14197fc20000, 0x7fac58f0?)\\n\\t/usr/lib/golang/src/fmt/print.go:674 +0x2f4 fp=0x14197fac58a8 sp=0x14197fac56e8 pc=0x1fdd1d4\\nfmt.(*pp).printArg(0x14197fc20000, {0x549a3a0, 0x14197fd92e80}, 0x76)\\n\\t/usr/lib/golang/src/fmt/print.go:757 +0x42e fp=0x14197fac5938 sp=0x14197fac58a8 pc=0x1fdd9ee\\nfmt.(*pp).doPrintf(0x14197fc20000, {0x553a400, 0x2}, {0x14197fac5d08, 0x1, 0x1})\\n\\t/usr/lib/golang/src/fmt/print.go:1075 +0x3ec fp=0x14197fac5a20 sp=0x14197fac5938 pc=0x1fe04ec\\nfmt.Sprintf({0x553a400, 0x2}, {0x14197fac5d08, 0x1, 0x1})\\n\\t/usr/lib/golang/src/fmt/print.go:239 +0x53 fp=0x14197fac5a78 sp=0x14197fac5a20 pc=0x1fda6b3\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197fc4b200}, 0x14197f1c02a0)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:160 +0x8d8 fp=0x14197fac5d60 sp=0x14197fac5a78 pc=0x391bad8\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x14197fdd8140?}, {0x5a00300?, 0x14197fc4b200?}, 0x0?)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x14197fac5e88 sp=0x14197fac5d60 pc=0x3929c16\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14197fac5f78 sp=0x14197fac5e88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14197fac5fe0 sp=0x14197fac5f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fac5fe8 sp=0x14197fac5fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 184 gp=0x14197fd3da40 m=nil [chan receive]:\\nruntime.gopark(0x14197f3c29a0?, 0x2034990?, 0x10?, 0xea?, 0x59f0ad0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14198011f9d8 sp=0x14198011f9b8 pc=0x1f38d2e\\nruntime.chanrecv(0x14197f3fa700, 0x14198011fcb8, 0x1)\\n\\t/usr/lib/golang/src/runtime/chan.go:667 +0x4ae fp=0x14198011fa50 sp=0x14198011f9d8 pc=0x1ec7d2e\\nruntime.chanrecv2(0x5a09ed0?, 0x14197f530f90?)\\n\\t/usr/lib/golang/src/runtime/chan.go:514 +0x12 fp=0x14198011fa78 sp=0x14198011fa50 pc=0x1ec7872\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x14197f696f20, {0x59f0b40, 0x14197fdd8140}, {0x5a00300, 0x14197fc4b340}, 0x14197f1c02a0)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:141 +0x94d fp=0x14198011fd60 sp=0x14198011fa78 pc=0x391bb4d\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x74666968736e6570?, {0x59f0b40?, 0x14197fdd8140?}, {0x5a00300?, 0x14197fc4b340?}, 0x6d736f63222c2273?)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x14198011fe88 sp=0x14198011fd60 pc=0x3929c16\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x14198011ff78 sp=0x14198011fe88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x14198011ffe0 sp=0x14198011ff78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14198011ffe8 sp=0x14198011ffe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 87\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 135 gp=0x14197fb465a0 m=nil [semacquire]:\\nruntime.gopark(0x14197f454000?, 0x22dc6c0?, 0xa0?, 0x82?, 0x1419808322a0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fe99ce8 sp=0x14197fe99cc8 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x14197fe99d50 sp=0x14197fe99ce8 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x14197fe99df0 sp=0x14197fe99d50 pc=0x1edbb12\\nruntime.gcAssistAlloc(0x14197fb465a0)\\n\\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x14197fe99e80 sp=0x14197fe99df0 pc=0x1ee06b6\\nruntime.deductAssistCredit(0x0?)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x14197fe99ea0 sp=0x14197fe99e80 pc=0x1ed10a6\\nruntime.mallocgc(0x70, 0x0, 0x1)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x14197fe99ed0 sp=0x14197fe99ea0 pc=0x1f35e07\\nruntime.makechan(0x0?, 0x0)\\n\\t/usr/lib/golang/src/runtime/chan.go:99 +0x126 fp=0x14197fe99f10 sp=0x14197fe99ed0 pc=0x1ec6826\\nnet/http.(*http2clientStream).cleanupWriteRequest(0x14197f2f1200, {0x0, 0x0})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8951 +0xd6 fp=0x14197fe99f90 sp=0x14197fe99f10 pc=0x22db1b6\\nnet/http.(*http2clientStream).doRequest(0x14197f2f1200, 0x14197f7480a0?, 0x14197f4d0020?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8719 +0x28 fp=0x14197fe99fb8 sp=0x14197fe99f90 pc=0x22d99e8\\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197fe99fe0 sp=0x14197fe99fb8 pc=0x22d999f\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fe99fe8 sp=0x14197fe99fe0 pc=0x1f40801\\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 110\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\\n\\ngoroutine 90 gp=0x14197fd82b40 m=nil [select]:\\nruntime.gopark(0x14197ff1df08?, 0x6?, 0x40?, 0x2b?, 0x14197ff1dda4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197f203c08 sp=0x14197f203be8 pc=0x1f38d2e\\nruntime.selectgo(0x14197f203f08, 0x14197ff1dd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197f203d38 sp=0x14197f203c08 pc=0x1f152a5\\nnet/http.(*http2clientStream).writeRequest(0x14197f2f0f00, 0x14197fb50140, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x14197f203f90 sp=0x14197f203d38 pc=0x22da6a8\\nnet/http.(*http2clientStream).doRequest(0x14197f2f0f00, 0x0?, 0x14197f1f1f98?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x14197f203fb8 sp=0x14197f203f90 pc=0x22d99d8\\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x14197f203fe0 sp=0x14197f203fb8 pc=0x22d999f\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197f203fe8 sp=0x14197f203fe0 pc=0x1f40801\\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 183\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\\n\\ngoroutine 225 gp=0x14197fd834a0 m=nil [sync.Cond.Wait]:\\nruntime.gopark(0x22b8a73?, 0x141980390000?, 0x0?, 0x60?, 0xe?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1419801298f8 sp=0x1419801298d8 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nsync.runtime_notifyListWait(0x14197f2f0f48, 0x3c)\\n\\t/usr/lib/golang/src/runtime/sema.go:617 +0x1b3 fp=0x141980129948 sp=0x1419801298f8 pc=0x1f3a8f3\\nsync.(*Cond).Wait(0x14197fbb1640?)\\n\\t/usr/lib/golang/src/sync/cond.go:71 +0x73 fp=0x141980129980 sp=0x141980129948 pc=0x1f4d673\\nnet/http.(*http2pipe).Read(0x14197f2f0f30, {0x14197fd98000, 0x1000, 0x1000})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:3928 +0xdb fp=0x1419801299e8 sp=0x141980129980 pc=0x22c2c1b\\nnet/http.http2transportResponseBody.Read({0x141980129a60?}, {0x14197fd98000?, 0x20a6c63?, 0x1f3ec12?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9852 +0x59 fp=0x141980129a70 sp=0x1419801299e8 pc=0x22dfdd9\\nbufio.(*Reader).fill(0x14197fc90180)\\n\\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x141980129aa8 sp=0x141980129a70 pc=0x20a6c63\\nbufio.(*Reader).ReadByte(0x14197fc90180)\\n\\t/usr/lib/golang/src/bufio/bufio.go:273 +0x27 fp=0x141980129ac0 sp=0x141980129aa8 pc=0x20a73c7\\ncompress/flate.(*decompressor).huffSym(0x141980058608, 0x141980058638)\\n\\t/usr/lib/golang/src/compress/flate/inflate.go:720 +0x7e fp=0x141980129b10 sp=0x141980129ac0 pc=0x228893e\\ncompress/flate.(*decompressor).huffmanBlock(0x82bed80?)\\n\\t/usr/lib/golang/src/compress/flate/inflate.go:495 +0x45 fp=0x141980129b90 sp=0x141980129b10 pc=0x2287605\\ncompress/flate.(*decompressor).Read(0x141980058608, {0x14197fd99cea, 0x316, 0xcea?})\\n\\t/usr/lib/golang/src/compress/flate/inflate.go:348 +0x5b fp=0x141980129bc0 sp=0x141980129b90 pc=0x2286dbb\\ncompress/gzip.(*Reader).Read(0x14197f3d8848, {0x14197fd99cea, 0x316, 0x316})\\n\\t/usr/lib/golang/src/compress/gzip/gunzip.go:252 +0x98 fp=0x141980129c50 sp=0x141980129bc0 pc=0x228c838\\ngithub.com/Azure/azure-kusto-go/azkustodata/internal/response.(*originalCloser).Read(0x141980129cf0?, {0x14197fd99cea?, 0x1000?, 0x0?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/internal/response/response.go:20 +0x22 fp=0x141980129c80 sp=0x141980129c50 pc=0x38b33e2\\nbufio.(*Reader).fill(0x14197f546110)\\n\\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x141980129cb8 sp=0x141980129c80 pc=0x20a6c63\\nbufio.(*Reader).ReadSlice(0x14197f546110, 0xa)\\n\\t/usr/lib/golang/src/bufio/bufio.go:377 +0x29 fp=0x141980129d00 sp=0x141980129cb8 pc=0x20a7769\\nbufio.(*Reader).collectFragments(0x14197f546110, 0xa)\\n\\t/usr/lib/golang/src/bufio/bufio.go:452 +0x85 fp=0x141980129de0 sp=0x141980129d00 pc=0x20a7b85\\nbufio.(*Reader).ReadBytes(0x4bbc5a0?, 0xa0?)\\n\\t/usr/lib/golang/src/bufio/bufio.go:479 +0x17 fp=0x141980129e60 sp=0x141980129de0 pc=0x20a7e77\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.(*frameReader).advance(0x14197f546100)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/frame_reader.go:64 +0x45 fp=0x141980129eb0 sp=0x141980129e60 pc=0x38d6d45\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readRoutine(0x14197f546100, 0x14197f8c54f0)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:85 +0x51 fp=0x141980129fc0 sp=0x141980129eb0 pc=0x38d7591\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap2()\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x1b fp=0x141980129fe0 sp=0x141980129fc0 pc=0x38d74db\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x141980129fe8 sp=0x141980129fe0 pc=0x1f40801\\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 183\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:74 +0x24a\\n\\ngoroutine 174 gp=0x14197ff64000 m=nil [select]:\\nruntime.gopark(0x1419800b9df8?, 0x2?, 0x0?, 0x0?, 0x1419800b9db4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x14197fcf3c50 sp=0x14197fcf3c30 pc=0x1f38d2e\\nruntime.selectgo(0x14197fcf3df8, 0x1419800b9db0, 0x59d87c8?, 0x0, 0x14197f2980e0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x14197fcf3d80 sp=0x14197fcf3c50 pc=0x1f152a5\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.nextFrame(0x14197f8c8370)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:198 +0x96 fp=0x14197fcf3e38 sp=0x14197fcf3d80 pc=0x38d7d16\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readPrimaryTable(0x14197f8c8370, 0x14197fb50000)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:258 +0xb1 fp=0x14197fcf3ec8 sp=0x14197fcf3e38 pc=0x38d8171\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x14197f8c8370)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:174 +0x189 fp=0x14197fcf3f30 sp=0x14197fcf3ec8 pc=0x38d7b89\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x14197f8c8370, 0x14197f59aee0)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x14197fcf3fc0 sp=0x14197fcf3f30 pc=0x38d788c\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x14197fcf3fe0 sp=0x14197fcf3fc0 pc=0x38d751b\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x14197fcf3fe8 sp=0x14197fcf3fe0 pc=0x1f40801\\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 184\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc\\n\\nrax    0x0\\nrbx    0x7f808bfff640\\nrcx    0x7f80df60594c\\nrdx    0x6\\nrdi    0x109fd\\nrsi    0x10a00\\nrbp    0x10a00\\nrsp    0x7f808bffe790\\nr8     0x7f808bffe860\\nr9     0x7f80df72b4e0\\nr10    0x8\\nr11    0x246\\nr12    0x6\\nr13    0x7f808bffeb60\\nr14    0x14197f1334a0\\nr15    0xffffffffffffffff\\nrip    0x7f80df60594c\\nrflags 0x246\\ncs     0x33\\nfs     0x0\\ngs     0x0\\n\\n  query-infra-svc: command failed: exit status 2\\noutput: runtime/cgo: pthread_create failed: Resource temporarily unavailable\\nSIGABRT: abort\\nPC=0x7f7ef9baa94c m=3 sigcode=18446744073709551610\\n\\ngoroutine 0 gp=0x170ac5c1f0e0 m=3 mp=0x170ac5c91808 [idle]:\\nruntime: g 0 gp=0x170ac5c1f0e0: unknown pc 0x7f7ef9baa94c\\nstack: frame={sp:0x7f7eabffe790, fp:0x0} stack=[0x7f7eab7ff1e0,0x7f7eabffede0)\\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\\n00007f7eabffe690: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6a0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6b0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6c0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6d0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6e0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6f0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe700: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe710: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe720: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe730: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe740: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe750: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe760: 00000000 00000000  00007f7e f9bb7fa9  ............~...\\n00007f7eabffe770: 00000000 00000000  00000000 00000011  ................\\n00007f7eabffe780: 00000000 00000130  00007f7e f9baa93e  0.......\u003e...~...\\n00007f7eabffe790: 00000000 00000120  00000000 00000000   ...............\\n                  ^ SP\\n00007f7eabffe7a0: 00000000 00000013  00000000 00000000  ................\\n00007f7eabffe7b0: 00000000 00000000  00007f7e f9c2dcff  ............~...\\n00007f7eabffe7c0: 00007f7e 7bfff910  00007f7e f9c2dcb1  ...{~.......~...\\n00007f7eabffe7d0: 00007f7e 7bfff640  00007f7e abffeaa0  @..{~.......~...\\n00007f7eabffe7e0: 00007f7e abffe90e  00007f7e abffe90f  ....~.......~...\\n00007f7eabffe7f0: 00007f7e 7bfff640  00007f7e f9ba87f5  @..{~.......~...\\n00007f7eabffe800: 00007f7e a4000020  00000000 00000120   ...~... .......\\n00007f7eabffe810: 00000000 003d0f00  176a34c4 bac87500  ..=......u...4j.\\n00007f7eabffe820: 00007f7e abfff640  00000000 00000006  @...~...........\\n00007f7eabffe830: 00007f7e a4000f30  00007f7e abffeb60  0...~...`...~...\\n00007f7eabffe840: 0000170a c5c1f0e0  00007f7e f9b5d646  ........F...~...\\n00007f7eabffe850: 00007f7e f9d1ae90  00007f7e f9b477f3  ....~....w..~...\\n00007f7eabffe860: 00000000 00000020  176a34c4 bac87500   ........u...4j.\\n00007f7eabffe870: 00007f7e 7b7ff000  00007f7e f9c1cac4  ...{~.......~...\\n00007f7eabffe880: 00000000 00000000  00000000 0000000d  ................\\nruntime: g 0 gp=0x170ac5c1f0e0: unknown pc 0x7f7ef9baa94c\\nstack: frame={sp:0x7f7eabffe790, fp:0x0} stack=[0x7f7eab7ff1e0,0x7f7eabffede0)\\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\\n00007f7eabffe690: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6a0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6b0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6c0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6d0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6e0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe6f0: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe700: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe710: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe720: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe730: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe740: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe750: 00000000 00000000  00000000 00000000  ................\\n00007f7eabffe760: 00000000 00000000  00007f7e f9bb7fa9  ............~...\\n00007f7eabffe770: 00000000 00000000  00000000 00000011  ................\\n00007f7eabffe780: 00000000 00000130  00007f7e f9baa93e  0.......\u003e...~...\\n00007f7eabffe790: 00000000 00000120  00000000 00000000   ...............\\n                  ^ SP\\n00007f7eabffe7a0: 00000000 00000013  00000000 00000000  ................\\n00007f7eabffe7b0: 00000000 00000000  00007f7e f9c2dcff  ............~...\\n00007f7eabffe7c0: 00007f7e 7bfff910  00007f7e f9c2dcb1  ...{~.......~...\\n00007f7eabffe7d0: 00007f7e 7bfff640  00007f7e abffeaa0  @..{~.......~...\\n00007f7eabffe7e0: 00007f7e abffe90e  00007f7e abffe90f  ....~.......~...\\n00007f7eabffe7f0: 00007f7e 7bfff640  00007f7e f9ba87f5  @..{~.......~...\\n00007f7eabffe800: 00007f7e a4000020  00000000 00000120   ...~... .......\\n00007f7eabffe810: 00000000 003d0f00  176a34c4 bac87500  ..=......u...4j.\\n00007f7eabffe820: 00007f7e abfff640  00000000 00000006  @...~...........\\n00007f7eabffe830: 00007f7e a4000f30  00007f7e abffeb60  0...~...`...~...\\n00007f7eabffe840: 0000170a c5c1f0e0  00007f7e f9b5d646  ........F...~...\\n00007f7eabffe850: 00007f7e f9d1ae90  00007f7e f9b477f3  ....~....w..~...\\n00007f7eabffe860: 00000000 00000020  176a34c4 bac87500   ........u...4j.\\n00007f7eabffe870: 00007f7e 7b7ff000  00007f7e f9c1cac4  ...{~.......~...\\n00007f7eabffe880: 00000000 00000000  00000000 0000000d  ................\\n\\ngoroutine 1 gp=0x170ac5c1e1e0 m=nil [sync.WaitGroup.Wait]:\\nruntime.gopark(0x170ac5cb27b8?, 0x2022a4d?, 0x70?, 0x8e?, 0x170ac5cb27e8?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cb27a0 sp=0x170ac5cb2780 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x170ac65f8210, 0x0, 0x1, 0x0, 0x19)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x170ac5cb2808 sp=0x170ac5cb27a0 pc=0x1f162b2\\nsync.runtime_SemacquireWaitGroup(0x170ac6709950?, 0xe0?)\\n\\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x170ac5cb2840 sp=0x170ac5cb2808 pc=0x1f3a66e\\nsync.(*WaitGroup).Wait(0x170ac65f8208)\\n\\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x170ac5cb2868 sp=0x170ac5cb2840 pc=0x1f4f445\\ngolang.org/x/sync/errgroup.(*Group).Wait(0x170ac65f8200)\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x170ac5cb2888 sp=0x170ac5cb2868 pc=0x362507e\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0}, {0x170ac626a1e0, 0x6, 0x6})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x170ac5cb2948 sp=0x170ac5cb2888 pc=0x3927e05\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).gatherInfraLogs(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:595 +0xadd fp=0x170ac5cb2d30 sp=0x170ac5cb2948 pc=0x392799d\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x170ac5fe11c0, {0x59f0b08, 0x170ac64108d0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:333 +0x533 fp=0x170ac5cb3908 sp=0x170ac5cb2d30 pc=0x3921bf3\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedInfraQueryOptions).RunInfra(0x170ac5c2c108, {0x59f0b08, 0x170ac64108d0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:83 +0x71f fp=0x170ac5cb3b30 sp=0x170ac5cb3908 pc=0x393887f\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawInfraQueryOptions).Run(0x0?, {0x59f0b08, 0x170ac64108d0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_options.go:109 +0x54 fp=0x170ac5cb3b58 sp=0x170ac5cb3b30 pc=0x3939134\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryInfraCommand.func1(0x170ac66f5508?, {0x170ac5d40960?, 0x4?, 0x553b0e2?})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:45 +0x28 fp=0x170ac5cb3b80 sp=0x170ac5cb3b58 pc=0x3938128\\ngithub.com/spf13/cobra.(*Command).execute(0x170ac66f5508, {0x170ac5e36100, 0xa, 0xa})\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x170ac5cb3d28 sp=0x170ac5cb3b80 pc=0x212e614\\ngithub.com/spf13/cobra.(*Command).ExecuteC(0x170ac6228608)\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x170ac5cb3e18 sp=0x170ac5cb3d28 pc=0x212efa5\\ngithub.com/spf13/cobra.(*Command).Execute(...)\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071\\ngithub.com/spf13/cobra.(*Command).ExecuteContext(...)\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064\\nmain.main()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x170ac5cb3f48 sp=0x170ac5cb3e18 pc=0x47ddd6a\\nruntime.main()\\n\\t/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x170ac5cb3fe0 sp=0x170ac5cb3f48 pc=0x1f01975\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cb3fe8 sp=0x170ac5cb3fe0 pc=0x1f40801\\n\\ngoroutine 2 gp=0x170ac5c1ed20 m=nil [force gc (idle)]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdefa8 sp=0x170ac5cdef88 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.forcegchelper()\\n\\t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x170ac5cdefe0 sp=0x170ac5cdefa8 pc=0x1f01c93\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdefe8 sp=0x170ac5cdefe0 pc=0x1f40801\\ncreated by runtime.init.7 in goroutine 1\\n\\t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\\n\\ngoroutine 18 gp=0x170ac5d843c0 m=nil [GC sweep wait]:\\nruntime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cda788 sp=0x170ac5cda768 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.bgsweep(0x170ac5d92000)\\n\\t/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x170ac5cda7c8 sp=0x170ac5cda788 pc=0x1ee9f11\\nruntime.gcenable.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x170ac5cda7e0 sp=0x170ac5cda7c8 pc=0x1edadd7\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cda7e8 sp=0x170ac5cda7e0 pc=0x1f40801\\ncreated by runtime.gcenable in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\\n\\ngoroutine 19 gp=0x170ac5d845a0 m=nil [GC scavenge wait]:\\nruntime.gopark(0x10000?, 0x5981cd8?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdaf78 sp=0x170ac5cdaf58 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.(*scavengerState).park(0x82979c0)\\n\\t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x170ac5cdafa8 sp=0x170ac5cdaf78 pc=0x1ee77e9\\nruntime.bgscavenge(0x170ac5d92000)\\n\\t/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x170ac5cdafc8 sp=0x170ac5cdafa8 pc=0x1ee7d99\\nruntime.gcenable.gowrap2()\\n\\t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x170ac5cdafe0 sp=0x170ac5cdafc8 pc=0x1edad97\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdafe8 sp=0x170ac5cdafe0 pc=0x1f40801\\ncreated by runtime.gcenable in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\\n\\ngoroutine 34 gp=0x170ac5e00000 m=nil [GOMAXPROCS updater (idle)]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e06788 sp=0x170ac5e06768 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.updateMaxProcsGoroutine()\\n\\t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x170ac5e067e0 sp=0x170ac5e06788 pc=0x1f10267\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e067e8 sp=0x170ac5e067e0 pc=0x1f40801\\ncreated by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\\n\\t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\\n\\ngoroutine 35 gp=0x170ac5e005a0 m=nil [finalizer wait]:\\nruntime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x170ac5cde601?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cde620 sp=0x170ac5cde600 pc=0x1f38d2e\\nruntime.runFinalizers()\\n\\t/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x170ac5cde7e0 sp=0x170ac5cde620 pc=0x1ed9cc7\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cde7e8 sp=0x170ac5cde7e0 pc=0x1f40801\\ncreated by runtime.createfing in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d\\n\\ngoroutine 36 gp=0x170ac5ff81e0 m=nil [cleanup wait]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e06f68 sp=0x170ac5e06f48 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.(*cleanupQueue).dequeue(0x82980e0)\\n\\t/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x170ac5e06fa0 sp=0x170ac5e06f68 pc=0x1ed6074\\nruntime.runCleanups()\\n\\t/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x170ac5e06fe0 sp=0x170ac5e06fa0 pc=0x1ed66e5\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e06fe8 sp=0x170ac5e06fe0 pc=0x1f40801\\ncreated by runtime.(*cleanupQueue).createGs in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5\\n\\ngoroutine 37 gp=0x170ac611bc20 m=nil [GC worker (idle)]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e07740 sp=0x170ac5e07720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5e077c8 sp=0x170ac5e07740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5e077e0 sp=0x170ac5e077c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e077e8 sp=0x170ac5e077e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 3 gp=0x170ac5c1f680 m=nil [runnable]:\\nruntime.gopark(0x125c26559f5b?, 0x3?, 0x94?, 0x85?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdf740 sp=0x170ac5cdf720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdf7c8 sp=0x170ac5cdf740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdf7e0 sp=0x170ac5cdf7c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdf7e8 sp=0x170ac5cdf7e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 20 gp=0x170ac5d84780 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125c265c3db1?, 0x3?, 0x92?, 0xcd?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdb740 sp=0x170ac5cdb720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdb7c8 sp=0x170ac5cdb740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdb7e0 sp=0x170ac5cdb7c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdb7e8 sp=0x170ac5cdb7e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 38 gp=0x170ac6134000 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125c26588b9e?, 0x1?, 0xf7?, 0xb4?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e07f40 sp=0x170ac5e07f20 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5e07fc8 sp=0x170ac5e07f40 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5e07fe0 sp=0x170ac5e07fc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e07fe8 sp=0x170ac5e07fe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 4 gp=0x170ac5c1f860 m=10 mp=0x170ac6280008 [GC mark termination]:\\nruntime.systemstack_switch()\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x170ac5cdfea0 sp=0x170ac5cdfe90 pc=0x1f3ebc8\\nruntime.forEachP(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:2112\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x170ac5cdff40 sp=0x170ac5cdfea0 pc=0x1edbc34\\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x170ac5cdffc8 sp=0x170ac5cdff40 pc=0x1edda88\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdffe0 sp=0x170ac5cdffc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdffe8 sp=0x170ac5cdffe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 21 gp=0x170ac5d84960 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125c265818cf?, 0x3?, 0xa5?, 0x9c?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdbf40 sp=0x170ac5cdbf20 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdbfc8 sp=0x170ac5cdbf40 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdbfe0 sp=0x170ac5cdbfc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdbfe8 sp=0x170ac5cdbfe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 22 gp=0x170ac5d84b40 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125c265846e7?, 0x3?, 0x77?, 0xb6?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdc740 sp=0x170ac5cdc720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdc7c8 sp=0x170ac5cdc740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdc7e0 sp=0x170ac5cdc7c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdc7e8 sp=0x170ac5cdc7e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 23 gp=0x170ac5d84d20 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125c26581951?, 0x3?, 0x91?, 0x58?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cdcf40 sp=0x170ac5cdcf20 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x170ac5e0b9d0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x170ac5cdcfc8 sp=0x170ac5cdcf40 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x170ac5cdcfe0 sp=0x170ac5cdcfc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5cdcfe8 sp=0x170ac5cdcfe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 39 gp=0x170ac648a3c0 m=nil [select, locked to thread]:\\nruntime.gopark(0x170ac5e097a8?, 0x2?, 0x1?, 0xa3?, 0x170ac5e09794?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5e09628 sp=0x170ac5e09608 pc=0x1f38d2e\\nruntime.selectgo(0x170ac5e097a8, 0x170ac5e09790, 0x0?, 0x0, 0x0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5e09758 sp=0x170ac5e09628 pc=0x1f152a5\\nruntime.ensureSigM.func1()\\n\\t/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x170ac5e097e0 sp=0x170ac5e09758 pc=0x1f328e8\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e097e8 sp=0x170ac5e097e0 pc=0x1f40801\\ncreated by runtime.ensureSigM in goroutine 1\\n\\t/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5\\n\\ngoroutine 24 gp=0x170ac62f4960 m=8 mp=0x170ac6180008 [syscall]:\\nruntime.notetsleepg(0x82bdea0, 0xffffffffffffffff)\\n\\t/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x170ac5e037a0 sp=0x170ac5e03778 pc=0x1ece509\\nos/signal.signal_recv()\\n\\t/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x170ac5e037c0 sp=0x170ac5e037a0 pc=0x1f3b298\\nos/signal.loop()\\n\\t/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x170ac5e037e0 sp=0x170ac5e037c0 pc=0x2030553\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5e037e8 sp=0x170ac5e037e0 pc=0x1f40801\\ncreated by os/signal.Notify.func1.1 in goroutine 1\\n\\t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\\n\\ngoroutine 50 gp=0x170ac6135a40 m=nil [select]:\\nruntime.gopark(0x170ac66e3798?, 0x2?, 0x80?, 0x6f?, 0x170ac66e378c?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac66e3628 sp=0x170ac66e3608 pc=0x1f38d2e\\nruntime.selectgo(0x170ac66e3798, 0x170ac66e3788, 0x0?, 0x0, 0x0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac66e3758 sp=0x170ac66e3628 pc=0x1f152a5\\nos/signal.NotifyContext.func1()\\n\\t/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x170ac66e37e0 sp=0x170ac66e3758 pc=0x202ff9c\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac66e37e8 sp=0x170ac66e37e0 pc=0x1f40801\\ncreated by os/signal.NotifyContext in goroutine 1\\n\\t/usr/lib/golang/src/os/signal/signal.go:291 +0x158\\n\\ngoroutine 102 gp=0x170ac61843c0 m=nil [select]:\\nruntime.gopark(0x170ac5cefc70?, 0x2?, 0xc8?, 0xf9?, 0x170ac5cefb6c?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5cef9f0 sp=0x170ac5cef9d0 pc=0x1f38d2e\\nruntime.selectgo(0x170ac5cefc70, 0x170ac5cefb68, 0x554857c?, 0x0, 0x226?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5cefb20 sp=0x170ac5cef9f0 pc=0x1f152a5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x170ac5c49130}, 0x170ac65fa070, 0x170ac6410cf0)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:283 +0x21b fp=0x170ac5ceff48 sp=0x170ac5cefb20 pc=0x39204fb\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x170ac5ceff78 sp=0x170ac5ceff48 pc=0x392800c\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5ceffe0 sp=0x170ac5ceff78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5ceffe8 sp=0x170ac5ceffe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 104 gp=0x170ac6184d20 m=nil [select]:\\nruntime.gopark(0x170ac6992ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69929d8?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6992848 sp=0x170ac6992828 pc=0x1f38d2e\\nruntime.selectgo(0x170ac6992ac8, 0x170ac69929d0, 0x55562e1?, 0x0, 0x1f6d6cf?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6992978 sp=0x170ac6992848 pc=0x1f152a5\\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac619fb80, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac6992b40 sp=0x170ac6992978 pc=0x22d92e5\\nnet/http.(*http2ClientConn).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac619fb80, {0x4c?, 0xfa?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac6992c80 sp=0x170ac6992b40 pc=0x22d50ad\\nnet/http.(*http2Transport).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac619fb80?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac6992cc0 sp=0x170ac6992c80 pc=0x22e3a9c\\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac619fa40)\\n\\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac6992ea0 sp=0x170ac6992cc0 pc=0x231037f\\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\\n\\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac6992ec0 sp=0x170ac6992ea0 pc=0x2321178\\nnet/http.send(0x170ac619fa40, {0x59a1ce0, 0x8175400}, {0x170ac6993138?, 0x1f3bac6?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69930b0 sp=0x170ac6992ec0 pc=0x22ad9ab\\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac619fa40, {0x170ac69931e8?, 0x1fdd787?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac6993148 sp=0x170ac69930b0 pc=0x22ad1d8\\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac619fa40)\\n\\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac6993338 sp=0x170ac6993148 pc=0x22af817\\nnet/http.(*Client).Do(...)\\n\\t/usr/lib/golang/src/net/http/client.go:592\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5c49450}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e34a0}, 0x170ac65f7ce0, {0x170ac65d9560, 0x112})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac6993630 sp=0x170ac6993338 pc=0x38de6bd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5c49450}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x170ac65f7c80, 0x0, {0x0, ...}, ...})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac6993888 sp=0x170ac6993630 pc=0x38ddecd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5c49450?}, 0xa8?, {0x170ac5db1a50?, 0x170ac6764a48?}, 0x3?, 0x2022a4d?)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac6993960 sp=0x170ac6993888 pc=0x38dd40b\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194700?}, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x170ac6764a48, 0x3, 0x4})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69939e0 sp=0x170ac6993960 pc=0x38e51e2\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194700}, {0x170ac5db1a50, 0xb}, 0x170ac6906a20, {0x0?, 0x2?, 0x1?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac6993a78 sp=0x170ac69939e0 pc=0x38e4eaf\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020500}, 0x170ac65fa000)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac6993d60 sp=0x170ac6993a78 pc=0x391b5c5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac5fe1880?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020500?}, 0x0?)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac6993e88 sp=0x170ac6993d60 pc=0x3929c16\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac6993f78 sp=0x170ac6993e88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac6993fe0 sp=0x170ac6993f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6993fe8 sp=0x170ac6993fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 101 gp=0x170ac61d4000 m=nil [sync.WaitGroup.Wait]:\\nruntime.gopark(0x170ac5de7e00?, 0x1ed0765?, 0xb0?, 0xfa?, 0xe?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de7da0 sp=0x170ac5de7d80 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x170ac65f8290, 0x0, 0x1, 0x0, 0x19)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x170ac5de7e08 sp=0x170ac5de7da0 pc=0x1f162b2\\nsync.runtime_SemacquireWaitGroup(0x170ac6709a70?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x170ac5de7e40 sp=0x170ac5de7e08 pc=0x1f3a66e\\nsync.(*WaitGroup).Wait(0x170ac65f8288)\\n\\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x170ac5de7e68 sp=0x170ac5de7e40 pc=0x1f4f445\\ngolang.org/x/sync/errgroup.(*Group).Wait(0x170ac65f8280)\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x170ac5de7e88 sp=0x170ac5de7e68 pc=0x362507e\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x170ac5fe1140, {0x59f0b40, 0x170ac5c49130}, {0x170ac626a1e0, 0x6, 0x170ac5cac1c0?}, 0x170ac65fa000)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x170ac5de7f00 sp=0x170ac5de7e88 pc=0x392976a\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x170ac5de7f78 sp=0x170ac5de7f00 pc=0x392808e\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5de7fe0 sp=0x170ac5de7f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de7fe8 sp=0x170ac5de7fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 67 gp=0x170ac62f5a40 m=nil [IO wait]:\\nruntime.gopark(0x170ac6497780?, 0x8?, 0x40?, 0x16?, 0xb?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de47b8 sp=0x170ac5de4798 pc=0x1f38d2e\\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x170ac5de47f0 sp=0x170ac5de47b8 pc=0x1ef9e57\\ninternal/poll.runtime_pollWait(0x7f7eb1670000, 0x72)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x170ac5de4810 sp=0x170ac5de47f0 pc=0x1f375c5\\ninternal/poll.(*pollDesc).wait(0x170ac620b480?, 0x170ac65a2000?, 0x0)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x170ac5de4838 sp=0x170ac5de4810 pc=0x1fbd167\\ninternal/poll.(*pollDesc).waitRead(...)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\\ninternal/poll.(*FD).Read(0x170ac620b480, {0x170ac65a2000, 0x3500, 0x3500})\\n\\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x170ac5de48d0 sp=0x170ac5de4838 pc=0x1fbe4ce\\nnet.(*netFD).Read(0x170ac620b480, {0x170ac65a2000?, 0x170ac5de4901?, 0x22a2e75?})\\n\\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x170ac5de4918 sp=0x170ac5de48d0 pc=0x20798c5\\nnet.(*conn).Read(0x170ac62823d8, {0x170ac65a2000?, 0x7f7eb047fc00?, 0x7f7ef9af2e00?})\\n\\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x170ac5de4960 sp=0x170ac5de4918 pc=0x208b7a5\\ncrypto/tls.(*atLeastReader).Read(0x170ac6708f30, {0x170ac65a2000?, 0x170ac5de4a48?, 0x1efcc28?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x170ac5de49a8 sp=0x170ac5de4960 pc=0x2222ddb\\nbytes.(*Buffer).ReadFrom(0x170ac605fb28, {0x59a3680, 0x170ac6708f30})\\n\\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x170ac5de4a00 sp=0x170ac5de49a8 pc=0x1fe9258\\ncrypto/tls.(*Conn).readFromUntil(0x170ac605f888, {0x59a32c0, 0x170ac62823d8}, 0x222764c?)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x170ac5de4a38 sp=0x170ac5de4a00 pc=0x2222fbe\\ncrypto/tls.(*Conn).readRecordOrCCS(0x170ac605f888, 0x0)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x170ac5de4c18 sp=0x170ac5de4a38 pc=0x222009b\\ncrypto/tls.(*Conn).readRecord(...)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:588\\ncrypto/tls.(*Conn).Read(0x170ac605f888, {0x170ac6029000, 0x1000, 0x22c34e0?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x170ac5de4c80 sp=0x170ac5de4c18 pc=0x22267c5\\nbufio.(*Reader).Read(0x170ac647aa20, {0x170ac6153624, 0x9, 0x22e100e?})\\n\\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0x170ac5de4cb8 sp=0x170ac5de4c80 pc=0x20a71f7\\nio.ReadAtLeast({0x59a1260, 0x170ac647aa20}, {0x170ac6153624, 0x9, 0x9}, 0x9)\\n\\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0x170ac5de4d00 sp=0x170ac5de4cb8 pc=0x1fb17ee\\nio.ReadFull(...)\\n\\t/usr/lib/golang/src/io/io.go:354\\nnet/http.http2readFrameHeader({0x170ac6153624, 0x9, 0x170a00000005?}, {0x59a1260?, 0x170ac647aa20?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:1805 +0x65 fp=0x170ac5de4d50 sp=0x170ac5de4d00 pc=0x22b9e25\\nnet/http.(*http2Framer).ReadFrameHeader(0x170ac61535e0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:2071 +0x6b fp=0x170ac5de4dd8 sp=0x170ac5de4d50 pc=0x22ba9eb\\nnet/http.(*http2Framer).ReadFrame(0x170ac61535e0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:2130 +0x18 fp=0x170ac5de4e18 sp=0x170ac5de4dd8 pc=0x22bb1b8\\nnet/http.(*http2clientConnReadLoop).run(0x170ac5de4fa8)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9550 +0xca fp=0x170ac5de4f60 sp=0x170ac5de4e18 pc=0x22de14a\\nnet/http.(*http2ClientConn).readLoop(0x170ac61d9880)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x170ac5de4fc8 sp=0x170ac5de4f60 pc=0x22dd532\\nnet/http.(*http2Transport).newClientConn.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x170ac5de4fe0 sp=0x170ac5de4fc8 pc=0x22d7057\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de4fe8 sp=0x170ac5de4fe0 pc=0x1f40801\\ncreated by net/http.(*http2Transport).newClientConn in goroutine 66\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5\\n\\ngoroutine 73 gp=0x170ac648ad20 m=nil [IO wait]:\\nruntime.gopark(0x170ac6497e40?, 0x1edb176?, 0x80?, 0xed?, 0xb?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de6808 sp=0x170ac5de67e8 pc=0x1f38d2e\\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x170ac5de6840 sp=0x170ac5de6808 pc=0x1ef9e57\\ninternal/poll.runtime_pollWait(0x7f7eb166fe00, 0x72)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x170ac5de6860 sp=0x170ac5de6840 pc=0x1f375c5\\ninternal/poll.(*pollDesc).wait(0x170ac60c4e80?, 0x170ac5ecd300?, 0x0)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x170ac5de6888 sp=0x170ac5de6860 pc=0x1fbd167\\ninternal/poll.(*pollDesc).waitRead(...)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\\ninternal/poll.(*FD).Read(0x170ac60c4e80, {0x170ac5ecd300, 0x1300, 0x1300})\\n\\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x170ac5de6920 sp=0x170ac5de6888 pc=0x1fbe4ce\\nnet.(*netFD).Read(0x170ac60c4e80, {0x170ac5ecd300?, 0x170ac5ecd30a?, 0x5?})\\n\\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x170ac5de6968 sp=0x170ac5de6920 pc=0x20798c5\\nnet.(*conn).Read(0x170ac5d82278, {0x170ac5ecd300?, 0x7f7eb047fc00?, 0x7f7ef9af2e00?})\\n\\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x170ac5de69b0 sp=0x170ac5de6968 pc=0x208b7a5\\ncrypto/tls.(*atLeastReader).Read(0x170ac67088e8, {0x170ac5ecd300?, 0x170ac5de6a98?, 0x1efcc28?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x170ac5de69f8 sp=0x170ac5de69b0 pc=0x2222ddb\\nbytes.(*Buffer).ReadFrom(0x170ac5dd02a8, {0x59a3680, 0x170ac67088e8})\\n\\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x170ac5de6a50 sp=0x170ac5de69f8 pc=0x1fe9258\\ncrypto/tls.(*Conn).readFromUntil(0x170ac5dd0008, {0x59a32c0, 0x170ac5d82278}, 0x222764c?)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x170ac5de6a88 sp=0x170ac5de6a50 pc=0x2222fbe\\ncrypto/tls.(*Conn).readRecordOrCCS(0x170ac5dd0008, 0x0)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x170ac5de6c68 sp=0x170ac5de6a88 pc=0x222009b\\ncrypto/tls.(*Conn).readRecord(...)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:588\\ncrypto/tls.(*Conn).Read(0x170ac5dd0008, {0x170ac6564000, 0x1000, 0x4e55540?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x170ac5de6cd0 sp=0x170ac5de6c68 pc=0x22267c5\\nnet/http.(*persistConn).Read(0x170ac5f73180, {0x170ac6564000?, 0x59a0ea0?, 0x81260e0?})\\n\\t/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x170ac5de6d30 sp=0x170ac5de6cd0 pc=0x2319d07\\nbufio.(*Reader).fill(0x170ac65537a0)\\n\\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x170ac5de6d68 sp=0x170ac5de6d30 pc=0x20a6c63\\nbufio.(*Reader).Peek(0x170ac65537a0, 0x1)\\n\\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x170ac5de6d80 sp=0x170ac5de6d68 pc=0x20a6df3\\nnet/http.(*persistConn).readLoop(0x170ac5f73180)\\n\\t/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x170ac5de6fc8 sp=0x170ac5de6d80 pc=0x231a812\\nnet/http.(*Transport).dialConn.gowrap2()\\n\\t/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x170ac5de6fe0 sp=0x170ac5de6fc8 pc=0x2319197\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de6fe8 sp=0x170ac5de6fe0 pc=0x1f40801\\ncreated by net/http.(*Transport).dialConn in goroutine 82\\n\\t/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac\\n\\ngoroutine 103 gp=0x170ac648af00 m=nil [select]:\\nruntime.gopark(0x170ac5de0d00?, 0x2?, 0xa0?, 0x6a?, 0x170ac5de0c74?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac5de0ac8 sp=0x170ac5de0aa8 pc=0x1f38d2e\\nruntime.selectgo(0x170ac5de0d00, 0x170ac5de0c70, 0x553bfdc?, 0x0, 0x5a48c80?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac5de0bf8 sp=0x170ac5de0ac8 pc=0x1f152a5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x0?, {0x59f0b40, 0x170ac5c49130}, 0x170ac65fa000, 0x170ac65fa070)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x170ac5de0f10 sp=0x170ac5de0bf8 pc=0x392836e\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x170ac5de0f78 sp=0x170ac5de0f10 pc=0x3927f3d\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac5de0fe0 sp=0x170ac5de0f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac5de0fe8 sp=0x170ac5de0fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 74 gp=0x170ac648b4a0 m=nil [select]:\\nruntime.gopark(0x170ac5df1f48?, 0x2?, 0x0?, 0xb6?, 0x170ac5df1ef4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6760d88 sp=0x170ac6760d68 pc=0x1f38d2e\\nruntime.selectgo(0x170ac6760f48, 0x170ac5df1ef0, 0x170ac68d6980?, 0x0, 0x170ac655e8d0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6760eb8 sp=0x170ac6760d88 pc=0x1f152a5\\nnet/http.(*persistConn).writeLoop(0x170ac5f73180)\\n\\t/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x170ac6760fc8 sp=0x170ac6760eb8 pc=0x231c426\\nnet/http.(*Transport).dialConn.gowrap3()\\n\\t/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x170ac6760fe0 sp=0x170ac6760fc8 pc=0x2319157\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6760fe8 sp=0x170ac6760fe0 pc=0x1f40801\\ncreated by net/http.(*Transport).dialConn in goroutine 82\\n\\t/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe\\n\\ngoroutine 105 gp=0x170ac61d43c0 m=nil [select]:\\nruntime.gopark(0x170ac699cac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac699c9d8?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac699c848 sp=0x170ac699c828 pc=0x1f38d2e\\nruntime.selectgo(0x170ac699cac8, 0x170ac699c9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac699c978 sp=0x170ac699c848 pc=0x1f152a5\\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e000, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac699cb40 sp=0x170ac699c978 pc=0x22d92e5\\nnet/http.(*http2ClientConn).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e000, {0x4c?, 0xfa?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac699cc80 sp=0x170ac699cb40 pc=0x22d50ad\\nnet/http.(*http2Transport).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e000?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac699ccc0 sp=0x170ac699cc80 pc=0x22e3a9c\\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac619fcc0)\\n\\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac699cea0 sp=0x170ac699ccc0 pc=0x231037f\\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\\n\\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac699cec0 sp=0x170ac699cea0 pc=0x2321178\\nnet/http.send(0x170ac619fcc0, {0x59a1ce0, 0x8175400}, {0x170ac699d138?, 0x1f3bac6?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac699d0b0 sp=0x170ac699cec0 pc=0x22ad9ab\\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac619fcc0, {0x170ac699d1e8?, 0x1fdd787?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac699d148 sp=0x170ac699d0b0 pc=0x22ad1d8\\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac619fcc0)\\n\\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac699d338 sp=0x170ac699d148 pc=0x22af817\\nnet/http.(*Client).Do(...)\\n\\t/usr/lib/golang/src/net/http/client.go:592\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e000}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e36b0}, 0x170ac65f7e00, {0x170ac65d9680, 0x118})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac699d630 sp=0x170ac699d338 pc=0x38de6bd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e000}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x170ac65f7da0, 0x0, {0x0, ...}, ...})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac699d888 sp=0x170ac699d630 pc=0x38ddecd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e000?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de1a48?}, 0x3?, 0x2022a4d?)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac699d960 sp=0x170ac699d888 pc=0x38dd40b\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194850?}, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x170ac5de1a48, 0x3, 0x4})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac699d9e0 sp=0x170ac699d960 pc=0x38e51e2\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194850}, {0x170ac5db1a50, 0xb}, 0x170ac6906b60, {0x0?, 0x2?, 0x0?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac699da78 sp=0x170ac699d9e0 pc=0x38e4eaf\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac60207c0}, 0x170ac65fa000)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac699dd60 sp=0x170ac699da78 pc=0x391b5c5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac60207c0?}, 0x0?)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac699de88 sp=0x170ac699dd60 pc=0x3929c16\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac699df78 sp=0x170ac699de88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac699dfe0 sp=0x170ac699df78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac699dfe8 sp=0x170ac699dfe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 107 gp=0x170ac61d45a0 m=nil [select]:\\nruntime.gopark(0x170ac69b2ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69b29d8?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69b2848 sp=0x170ac69b2828 pc=0x1f38d2e\\nruntime.selectgo(0x170ac69b2ac8, 0x170ac69b29d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69b2978 sp=0x170ac69b2848 pc=0x1f152a5\\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e500, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69b2b40 sp=0x170ac69b2978 pc=0x22d92e5\\nnet/http.(*http2ClientConn).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e500, {0x4c?, 0xfa?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69b2c80 sp=0x170ac69b2b40 pc=0x22d50ad\\nnet/http.(*http2Transport).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e500?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69b2cc0 sp=0x170ac69b2c80 pc=0x22e3a9c\\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac699e3c0)\\n\\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69b2ea0 sp=0x170ac69b2cc0 pc=0x231037f\\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\\n\\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69b2ec0 sp=0x170ac69b2ea0 pc=0x2321178\\nnet/http.send(0x170ac699e3c0, {0x59a1ce0, 0x8175400}, {0x170ac69b3138?, 0x1f3bac6?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69b30b0 sp=0x170ac69b2ec0 pc=0x22ad9ab\\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac699e3c0, {0x4d3aa8?, 0xcd6610?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69b3148 sp=0x170ac69b30b0 pc=0x22ad1d8\\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac699e3c0)\\n\\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69b3338 sp=0x170ac69b3148 pc=0x22af817\\nnet/http.(*Client).Do(...)\\n\\t/usr/lib/golang/src/net/http/client.go:592\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e1e0}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3a60}, 0x170ac69a0120, {0x170ac65d98c0, 0x111})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69b3630 sp=0x170ac69b3338 pc=0x38de6bd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e1e0}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x170ac69a00c0, 0x0, {0x0, ...}, ...})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69b3888 sp=0x170ac69b3630 pc=0x38ddecd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e1e0?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de2a48?}, 0x3?, 0x2022a4d?)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69b3960 sp=0x170ac69b3888 pc=0x38dd40b\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194bd0?}, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x170ac5de2a48, 0x3, 0x4})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69b39e0 sp=0x170ac69b3960 pc=0x38e51e2\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194bd0}, {0x170ac5db1a50, 0xb}, 0x170ac6906de0, {0x0?, 0x2?, 0x0?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69b3a78 sp=0x170ac69b39e0 pc=0x38e4eaf\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020ec0}, 0x170ac65fa000)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69b3d60 sp=0x170ac69b3a78 pc=0x391b5c5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020ec0?}, 0x0?)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69b3e88 sp=0x170ac69b3d60 pc=0x3929c16\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69b3f78 sp=0x170ac69b3e88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69b3fe0 sp=0x170ac69b3f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69b3fe8 sp=0x170ac69b3fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 106 gp=0x170ac648b680 m=nil [select]:\\nruntime.gopark(0x170ac69aaac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69aa9d8?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69aa848 sp=0x170ac69aa828 pc=0x1f38d2e\\nruntime.selectgo(0x170ac69aaac8, 0x170ac69aa9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69aa978 sp=0x170ac69aa848 pc=0x1f152a5\\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e280, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69aab40 sp=0x170ac69aa978 pc=0x22d92e5\\nnet/http.(*http2ClientConn).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e280, {0x4c?, 0xfa?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69aac80 sp=0x170ac69aab40 pc=0x22d50ad\\nnet/http.(*http2Transport).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e280?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69aacc0 sp=0x170ac69aac80 pc=0x22e3a9c\\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac699e140)\\n\\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69aaea0 sp=0x170ac69aacc0 pc=0x231037f\\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\\n\\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69aaec0 sp=0x170ac69aaea0 pc=0x2321178\\nnet/http.send(0x170ac699e140, {0x59a1ce0, 0x8175400}, {0x170ac69ab138?, 0x1f3bac6?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69ab0b0 sp=0x170ac69aaec0 pc=0x22ad9ab\\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac699e140, {0x170ac69ab1e8?, 0x1fdd787?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69ab148 sp=0x170ac69ab0b0 pc=0x22ad1d8\\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac699e140)\\n\\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69ab338 sp=0x170ac69ab148 pc=0x22af817\\nnet/http.(*Client).Do(...)\\n\\t/usr/lib/golang/src/net/http/client.go:592\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e0f0}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e38c0}, 0x170ac69a0030, {0x170ac65d97a0, 0x10e})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69ab630 sp=0x170ac69ab338 pc=0x38de6bd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e0f0}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x170ac65f7ec0, 0x0, {0x0, ...}, ...})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69ab888 sp=0x170ac69ab630 pc=0x38ddecd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e0f0?}, 0xa8?, {0x170ac5db1a50?, 0x170ac6766a48?}, 0x3?, 0x2022a4d?)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69ab960 sp=0x170ac69ab888 pc=0x38dd40b\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194a80?}, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x170ac6766a48, 0x3, 0x4})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69ab9e0 sp=0x170ac69ab960 pc=0x38e51e2\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194a80}, {0x170ac5db1a50, 0xb}, 0x170ac6906ca0, {0x0?, 0x2?, 0x0?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69aba78 sp=0x170ac69ab9e0 pc=0x38e4eaf\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6020cc0}, 0x170ac65fa000)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69abd60 sp=0x170ac69aba78 pc=0x391b5c5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac6766ea0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6020cc0?}, 0x170ac5f6f3b0?)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69abe88 sp=0x170ac69abd60 pc=0x3929c16\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69abf78 sp=0x170ac69abe88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69abfe0 sp=0x170ac69abf78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69abfe8 sp=0x170ac69abfe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 108 gp=0x170ac61d4780 m=nil [select]:\\nruntime.gopark(0x170ac69c0ac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69c09d8?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69c0848 sp=0x170ac69c0828 pc=0x1f38d2e\\nruntime.selectgo(0x170ac69c0ac8, 0x170ac69c09d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69c0978 sp=0x170ac69c0848 pc=0x1f152a5\\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699e780, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69c0b40 sp=0x170ac69c0978 pc=0x22d92e5\\nnet/http.(*http2ClientConn).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699e780, {0x4c?, 0xfa?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69c0c80 sp=0x170ac69c0b40 pc=0x22d50ad\\nnet/http.(*http2Transport).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699e780?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69c0cc0 sp=0x170ac69c0c80 pc=0x22e3a9c\\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac699e640)\\n\\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69c0ea0 sp=0x170ac69c0cc0 pc=0x231037f\\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\\n\\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69c0ec0 sp=0x170ac69c0ea0 pc=0x2321178\\nnet/http.send(0x170ac699e640, {0x59a1ce0, 0x8175400}, {0x170ac69c1138?, 0x1f3bac6?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69c10b0 sp=0x170ac69c0ec0 pc=0x22ad9ab\\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac699e640, {0x82bed80?, 0x6401fdd700?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69c1148 sp=0x170ac69c10b0 pc=0x22ad1d8\\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac699e640)\\n\\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69c1338 sp=0x170ac69c1148 pc=0x22af817\\nnet/http.(*Client).Do(...)\\n\\t/usr/lib/golang/src/net/http/client.go:592\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e280}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3bf0}, 0x170ac69a0210, {0x170ac65d99e0, 0x110})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69c1630 sp=0x170ac69c1338 pc=0x38de6bd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e280}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x170ac69a01b0, 0x0, {0x0, ...}, ...})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69c1888 sp=0x170ac69c1630 pc=0x38ddecd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e280?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de2a48?}, 0x3?, 0x2022a4d?)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69c1960 sp=0x170ac69c1888 pc=0x38dd40b\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194d90?}, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x170ac5de2a48, 0x3, 0x4})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69c19e0 sp=0x170ac69c1960 pc=0x38e51e2\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194d90}, {0x170ac5db1a50, 0xb}, 0x170ac6906f20, {0x0?, 0x2?, 0x0?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69c1a78 sp=0x170ac69c19e0 pc=0x38e4eaf\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac6021d80}, 0x170ac65fa000)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69c1d60 sp=0x170ac69c1a78 pc=0x391b5c5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x170ac66e1d6a?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac6021d80?}, 0x0?)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69c1e88 sp=0x170ac69c1d60 pc=0x3929c16\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69c1f78 sp=0x170ac69c1e88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69c1fe0 sp=0x170ac69c1f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69c1fe8 sp=0x170ac69c1fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 109 gp=0x170ac61d4960 m=nil [select]:\\nruntime.gopark(0x170ac69caac8?, 0x4?, 0xa0?, 0x6a?, 0x170ac69ca9d8?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac69ca848 sp=0x170ac69ca828 pc=0x1f38d2e\\nruntime.selectgo(0x170ac69caac8, 0x170ac69ca9d0, 0x55562e1?, 0x0, 0x1ebb329?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac69ca978 sp=0x170ac69ca848 pc=0x1f152a5\\nnet/http.(*http2ClientConn).roundTrip(0x170ac61d9880, 0x170ac699ea00, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8688 +0x525 fp=0x170ac69cab40 sp=0x170ac69ca978 pc=0x22d92e5\\nnet/http.(*http2ClientConn).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8603\\nnet/http.(*http2Transport).RoundTripOpt(0x170ac627c960, 0x170ac699ea00, {0x4c?, 0xfa?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7901 +0x1ed fp=0x170ac69cac80 sp=0x170ac69cab40 pc=0x22d50ad\\nnet/http.(*http2Transport).RoundTrip(...)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:7852\\nnet/http.http2noDialH2RoundTripper.RoundTrip({0x8175400?}, 0x170ac699ea00?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:10560 +0x1c fp=0x170ac69cacc0 sp=0x170ac69cac80 pc=0x22e3a9c\\nnet/http.(*Transport).roundTrip(0x8175400, 0x170ac699e8c0)\\n\\t/usr/lib/golang/src/net/http/transport.go:623 +0x4ff fp=0x170ac69caea0 sp=0x170ac69cacc0 pc=0x231037f\\nnet/http.(*Transport).RoundTrip(0x4f184e0?, 0x59a1ce0?)\\n\\t/usr/lib/golang/src/net/http/roundtrip.go:33 +0x18 fp=0x170ac69caec0 sp=0x170ac69caea0 pc=0x2321178\\nnet/http.send(0x170ac699e8c0, {0x59a1ce0, 0x8175400}, {0x170ac69cb138?, 0x1f3bac6?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:264 +0x64b fp=0x170ac69cb0b0 sp=0x170ac69caec0 pc=0x22ad9ab\\nnet/http.(*Client).send(0x170ac6410c60, 0x170ac699e8c0, {0x170ac69cb1e8?, 0x1fdd787?, 0x0?})\\n\\t/usr/lib/golang/src/net/http/client.go:185 +0x258 fp=0x170ac69cb148 sp=0x170ac69cb0b0 pc=0x22ad1d8\\nnet/http.(*Client).do(0x170ac6410c60, 0x170ac699e8c0)\\n\\t/usr/lib/golang/src/net/http/client.go:733 +0x9d7 fp=0x170ac69cb338 sp=0x170ac69cb148 pc=0x22af817\\nnet/http.(*Client).Do(...)\\n\\t/usr/lib/golang/src/net/http/client.go:592\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequestImpl(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e410}, 0x1, 0x170ac5e75950, {0x59d1870, 0x170ac65e3d80}, 0x170ac69a0300, {0x170ac65d9b00, 0x114})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:192 +0x61d fp=0x170ac69cb630 sp=0x170ac69cb338 pc=0x38de6bd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).doRequest(0x170ac604e5f0, {0x59f0b40, 0x170ac5d5e410}, 0x1, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x170ac69a02a0, 0x0, {0x0, ...}, ...})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:149 +0xa2d fp=0x170ac69cb888 sp=0x170ac69cb630 pc=0x38ddecd\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Conn).rawQuery(0x59f0b40?, {0x59f0b40?, 0x170ac5d5e410?}, 0xa8?, {0x170ac5db1a50?, 0x170ac5de3a48?}, 0x3?, 0x2022a4d?)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/conn.go:86 +0xab fp=0x170ac69cb960 sp=0x170ac69cb888 pc=0x38dd40b\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).rawV2(0x170ac5fe10c0, {0x59f0bb0?, 0x170ac6194620?}, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x170ac5de3a48, 0x3, 0x4})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:191 +0x102 fp=0x170ac69cb9e0 sp=0x170ac69cb960 pc=0x38e51e2\\ngithub.com/Azure/azure-kusto-go/azkustodata.(*Client).IterativeQuery(0x170ac5fe10c0, {0x59f0bb0, 0x170ac6194620}, {0x170ac5db1a50, 0xb}, 0x170ac6907060, {0x0?, 0x2?, 0x2700000000000000?})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/kusto.go:147 +0x24f fp=0x170ac69cba78 sp=0x170ac69cb9e0 pc=0x38e4eaf\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x170ac5e1f2a0, {0x59f0b40, 0x170ac5c49220}, {0x5a00300, 0x170ac65f81c0}, 0x170ac65fa000)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:113 +0x3c5 fp=0x170ac69cbd60 sp=0x170ac69cba78 pc=0x391b5c5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x0?, {0x59f0b40?, 0x170ac5c49220?}, {0x5a00300?, 0x170ac65f81c0?}, 0x0?)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x170ac69cbe88 sp=0x170ac69cbd60 pc=0x3929c16\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x170ac69cbf78 sp=0x170ac69cbe88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x170ac69cbfe0 sp=0x170ac69cbf78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac69cbfe8 sp=0x170ac69cbfe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 101\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 110 gp=0x170ac61d4b40 m=nil [select]:\\nruntime.gopark(0x170ac6927f08?, 0x6?, 0x40?, 0x4b?, 0x170ac6927da4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6927c08 sp=0x170ac6927be8 pc=0x1f38d2e\\nruntime.selectgo(0x170ac6927f08, 0x170ac6927d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6927d38 sp=0x170ac6927c08 pc=0x1f152a5\\nnet/http.(*http2clientStream).writeRequest(0x170ac61a0f00, 0x170ac619fb80, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6927f90 sp=0x170ac6927d38 pc=0x22da6a8\\nnet/http.(*http2clientStream).doRequest(0x170ac61a0f00, 0x0?, 0x170ac66e2798?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6927fb8 sp=0x170ac6927f90 pc=0x22d99d8\\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6927fe0 sp=0x170ac6927fb8 pc=0x22d999f\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6927fe8 sp=0x170ac6927fe0 pc=0x1f40801\\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 104\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\\n\\ngoroutine 111 gp=0x170ac61d4d20 m=nil [select]:\\nruntime.gopark(0x170ac658df08?, 0x6?, 0x0?, 0x7?, 0x170ac658dda4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac658dc08 sp=0x170ac658dbe8 pc=0x1f38d2e\\nruntime.selectgo(0x170ac658df08, 0x170ac658dd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac658dd38 sp=0x170ac658dc08 pc=0x1f152a5\\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1080, 0x170ac699e000, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac658df90 sp=0x170ac658dd38 pc=0x22da6a8\\nnet/http.(*http2clientStream).doRequest(0x170ac61a1080, 0x0?, 0x170ac66e2798?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac658dfb8 sp=0x170ac658df90 pc=0x22d99d8\\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac658dfe0 sp=0x170ac658dfb8 pc=0x22d999f\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac658dfe8 sp=0x170ac658dfe0 pc=0x1f40801\\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 105\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\\n\\ngoroutine 112 gp=0x170ac61d4f00 m=nil [select]:\\nruntime.gopark(0x170ac6589f08?, 0x6?, 0x0?, 0x7?, 0x170ac6589da4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6589c08 sp=0x170ac6589be8 pc=0x1f38d2e\\nruntime.selectgo(0x170ac6589f08, 0x170ac6589d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6589d38 sp=0x170ac6589c08 pc=0x1f152a5\\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1200, 0x170ac699e280, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6589f90 sp=0x170ac6589d38 pc=0x22da6a8\\nnet/http.(*http2clientStream).doRequest(0x170ac61a1200, 0x0?, 0x170ac66e1f98?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6589fb8 sp=0x170ac6589f90 pc=0x22d99d8\\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6589fe0 sp=0x170ac6589fb8 pc=0x22d999f\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6589fe8 sp=0x170ac6589fe0 pc=0x1f40801\\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 106\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\\n\\ngoroutine 113 gp=0x170ac61d50e0 m=nil [select]:\\nruntime.gopark(0x170ac658ff08?, 0x6?, 0x0?, 0x7?, 0x170ac658fda4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac658fc08 sp=0x170ac658fbe8 pc=0x1f38d2e\\nruntime.selectgo(0x170ac658ff08, 0x170ac658fd98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac658fd38 sp=0x170ac658fc08 pc=0x1f152a5\\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1380, 0x170ac699e780, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac658ff90 sp=0x170ac658fd38 pc=0x22da6a8\\nnet/http.(*http2clientStream).doRequest(0x170ac61a1380, 0x0?, 0x170ac66e1f98?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac658ffb8 sp=0x170ac658ff90 pc=0x22d99d8\\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac658ffe0 sp=0x170ac658ffb8 pc=0x22d999f\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac658ffe8 sp=0x170ac658ffe0 pc=0x1f40801\\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 108\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\\n\\ngoroutine 114 gp=0x170ac61d52c0 m=nil [select]:\\nruntime.gopark(0x170ac6923f08?, 0x6?, 0x0?, 0x7?, 0x170ac6923da4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6923c08 sp=0x170ac6923be8 pc=0x1f38d2e\\nruntime.selectgo(0x170ac6923f08, 0x170ac6923d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6923d38 sp=0x170ac6923c08 pc=0x1f152a5\\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1500, 0x170ac699ea00, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6923f90 sp=0x170ac6923d38 pc=0x22da6a8\\nnet/http.(*http2clientStream).doRequest(0x170ac61a1500, 0x0?, 0x170ac66e1f98?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6923fb8 sp=0x170ac6923f90 pc=0x22d99d8\\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6923fe0 sp=0x170ac6923fb8 pc=0x22d999f\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6923fe8 sp=0x170ac6923fe0 pc=0x1f40801\\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 109\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\\n\\ngoroutine 115 gp=0x170ac61d54a0 m=nil [select]:\\nruntime.gopark(0x170ac6925f08?, 0x6?, 0x38?, 0x5b?, 0x170ac6925da4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x170ac6925c08 sp=0x170ac6925be8 pc=0x1f38d2e\\nruntime.selectgo(0x170ac6925f08, 0x170ac6925d98, 0x553dcb1?, 0x0, 0x3929c16?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x170ac6925d38 sp=0x170ac6925c08 pc=0x1f152a5\\nnet/http.(*http2clientStream).writeRequest(0x170ac61a1680, 0x170ac699e500, 0x0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8857 +0xc88 fp=0x170ac6925f90 sp=0x170ac6925d38 pc=0x22da6a8\\nnet/http.(*http2clientStream).doRequest(0x170ac61a1680, 0x0?, 0x170ac66e1f98?)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8718 +0x18 fp=0x170ac6925fb8 sp=0x170ac6925f90 pc=0x22d99d8\\nnet/http.(*http2ClientConn).roundTrip.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x1f fp=0x170ac6925fe0 sp=0x170ac6925fb8 pc=0x22d999f\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x170ac6925fe8 sp=0x170ac6925fe0 pc=0x1f40801\\ncreated by net/http.(*http2ClientConn).roundTrip in goroutine 107\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8624 +0x470\\n\\nrax    0x0\\nrbx    0x7f7eabfff640\\nrcx    0x7f7ef9baa94c\\nrdx    0x6\\nrdi    0x10a2e\\nrsi    0x10a30\\nrbp    0x10a30\\nrsp    0x7f7eabffe790\\nr8     0x7f7eabffe860\\nr9     0x7f7ef9cd04e0\\nr10    0x8\\nr11    0x246\\nr12    0x6\\nr13    0x7f7eabffeb60\\nr14    0x170ac5c1f0e0\\nr15    0xffffffffffffffff\\nrip    0x7f7ef9baa94c\\nrflags 0x246\\ncs     0x33\\nfs     0x0\\ngs     0x0\\n\\n  query-infra-both: command failed: exit status 2\\noutput: runtime/cgo: pthread_create failed: Resource temporarily unavailable\\nSIGABRT: abort\\nPC=0x7f5e9c16994c m=2 sigcode=18446744073709551610\\n\\ngoroutine 0 gp=0x280029a48b40 m=2 mp=0x280029abb008 [idle]:\\nruntime: g 0 gp=0x280029a48b40: unknown pc 0x7f5e9c16994c\\nstack: frame={sp:0x7f5e554a7740, fp:0x0} stack=[0x7f5e54ca81e0,0x7f5e554a7de0)\\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\\n00007f5e554a7640: 00000000 01eb64e5  00000000 00000000  .d..............\\n                  ^ \u003cinternal/runtime/syscall/linux.EpollWait+0x45\u003e\\n00007f5e554a7650: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7660: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7670: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7680: 00007f5e 554a76d4  00007f5e 554a7d18  .vJU^....}JU^...\\n00007f5e554a7690: 00000000 01efa894  00000000 00000000  ................\\n                  ^ \u003cruntime.netpoll+0xd4\u003e\\n00007f5e554a76a0: 00007f5e 554a76d4  00000000 00000000  .vJU^...........\\n00007f5e554a76b0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a76c0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a76d0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a76e0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a76f0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7700: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7710: 00000000 00000000  00007f5e 9c176fa9  .........o..^...\\n00007f5e554a7720: 00000000 00000000  00000000 00000011  ................\\n00007f5e554a7730: 00000000 00000130  00007f5e 9c16993e  0.......\u003e...^...\\n00007f5e554a7740: 00000000 00000120  00000000 00000000   ...............\\n                  ^ SP\\n00007f5e554a7750: 00000000 00000013  00000000 00000000  ................\\n00007f5e554a7760: 00000000 00000000  00007f5e 9c1eccff  ............^...\\n00007f5e554a7770: 00007f5e 327fc910  00007f5e 9c1eccb1  ...2^.......^...\\n00007f5e554a7780: 00007f5e 327fc640  00007f5e 554a7a50  @..2^...PzJU^...\\n00007f5e554a7790: 00007f5e 554a78be  00007f5e 554a78bf  .xJU^....xJU^...\\n00007f5e554a77a0: 00007f5e 327fc640  00007f5e 9c1677f5  @..2^....w..^...\\n00007f5e554a77b0: 00007f5e 50000020  00000000 00000120   ..P^... .......\\n00007f5e554a77c0: 00000000 003d0f00  5a23f32c fe4c8400  ..=.......L.,.#Z\\n00007f5e554a77d0: 00007f5e 554a8640  00000000 00000006  @.JU^...........\\n00007f5e554a77e0: 00000000 0bac42a0  00007f5e 554a7b10  .B.......{JU^...\\n00007f5e554a77f0: 00002800 29a48b40  00007f5e 9c11c646  @..).(..F...^...\\n00007f5e554a7800: 00007f5e 9c2d9e90  00007f5e 9c1067f3  ..-.^....g..^...\\n00007f5e554a7810: 00000000 00000020  5a23f32c fe4c8400   .........L.,.#Z\\n00007f5e554a7820: 00007f5e 31ffc000  00007f5e 9c1dbac4  ...1^.......^...\\n00007f5e554a7830: 00000000 00000000  00000000 0000000d  ................\\nruntime: g 0 gp=0x280029a48b40: unknown pc 0x7f5e9c16994c\\nstack: frame={sp:0x7f5e554a7740, fp:0x0} stack=[0x7f5e54ca81e0,0x7f5e554a7de0)\\n                   7 6 5 4  3 2 1 0   f e d c  b a 9 8  0123456789abcdef\\n00007f5e554a7640: 00000000 01eb64e5  00000000 00000000  .d..............\\n                  ^ \u003cinternal/runtime/syscall/linux.EpollWait+0x45\u003e\\n00007f5e554a7650: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7660: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7670: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7680: 00007f5e 554a76d4  00007f5e 554a7d18  .vJU^....}JU^...\\n00007f5e554a7690: 00000000 01efa894  00000000 00000000  ................\\n                  ^ \u003cruntime.netpoll+0xd4\u003e\\n00007f5e554a76a0: 00007f5e 554a76d4  00000000 00000000  .vJU^...........\\n00007f5e554a76b0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a76c0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a76d0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a76e0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a76f0: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7700: 00000000 00000000  00000000 00000000  ................\\n00007f5e554a7710: 00000000 00000000  00007f5e 9c176fa9  .........o..^...\\n00007f5e554a7720: 00000000 00000000  00000000 00000011  ................\\n00007f5e554a7730: 00000000 00000130  00007f5e 9c16993e  0.......\u003e...^...\\n00007f5e554a7740: 00000000 00000120  00000000 00000000   ...............\\n                  ^ SP\\n00007f5e554a7750: 00000000 00000013  00000000 00000000  ................\\n00007f5e554a7760: 00000000 00000000  00007f5e 9c1eccff  ............^...\\n00007f5e554a7770: 00007f5e 327fc910  00007f5e 9c1eccb1  ...2^.......^...\\n00007f5e554a7780: 00007f5e 327fc640  00007f5e 554a7a50  @..2^...PzJU^...\\n00007f5e554a7790: 00007f5e 554a78be  00007f5e 554a78bf  .xJU^....xJU^...\\n00007f5e554a77a0: 00007f5e 327fc640  00007f5e 9c1677f5  @..2^....w..^...\\n00007f5e554a77b0: 00007f5e 50000020  00000000 00000120   ..P^... .......\\n00007f5e554a77c0: 00000000 003d0f00  5a23f32c fe4c8400  ..=.......L.,.#Z\\n00007f5e554a77d0: 00007f5e 554a8640  00000000 00000006  @.JU^...........\\n00007f5e554a77e0: 00000000 0bac42a0  00007f5e 554a7b10  .B.......{JU^...\\n00007f5e554a77f0: 00002800 29a48b40  00007f5e 9c11c646  @..).(..F...^...\\n00007f5e554a7800: 00007f5e 9c2d9e90  00007f5e 9c1067f3  ..-.^....g..^...\\n00007f5e554a7810: 00000000 00000020  5a23f32c fe4c8400   .........L.,.#Z\\n00007f5e554a7820: 00007f5e 31ffc000  00007f5e 9c1dbac4  ...1^.......^...\\n00007f5e554a7830: 00000000 00000000  00000000 0000000d  ................\\n\\ngoroutine 1 gp=0x280029a481e0 m=nil [sync.WaitGroup.Wait]:\\nruntime.gopark(0x28002a5927b8?, 0x2022a4d?, 0x30?, 0xe9?, 0x28002a5927e8?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a5927a0 sp=0x28002a592780 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x280029ae34d0, 0x0, 0x1, 0x0, 0x19)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a592808 sp=0x28002a5927a0 pc=0x1f162b2\\nsync.runtime_SemacquireWaitGroup(0x28002a83b578?, 0xe0?)\\n\\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x28002a592840 sp=0x28002a592808 pc=0x1f3a66e\\nsync.(*WaitGroup).Wait(0x280029ae34c8)\\n\\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x28002a592868 sp=0x28002a592840 pc=0x1f4f445\\ngolang.org/x/sync/errgroup.(*Group).Wait(0x280029ae34c0)\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x28002a592888 sp=0x28002a592868 pc=0x362507e\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0}, {0x28002a977b50, 0x1, 0x1})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:654 +0x3a5 fp=0x28002a592948 sp=0x28002a592888 pc=0x3927e05\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).gatherInfraLogs(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:587 +0x867 fp=0x28002a592d30 sp=0x28002a592948 pc=0x3927727\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).GatherLogs(0x28002a7c7840, {0x59f0b08, 0x28002a1873b0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:333 +0x533 fp=0x28002a593908 sp=0x28002a592d30 pc=0x3921bf3\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*CompletedInfraQueryOptions).RunInfra(0x28002a11e1e0, {0x59f0b08, 0x28002a1873b0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:83 +0x71f fp=0x28002a593b30 sp=0x28002a593908 pc=0x393887f\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.(*RawInfraQueryOptions).Run(0x0?, {0x59f0b08, 0x28002a1873b0})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_options.go:109 +0x54 fp=0x28002a593b58 sp=0x28002a593b30 pc=0x3939134\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather.newQueryInfraCommand.func1(0x28002a53d508?, {0x280029ca3b00?, 0x4?, 0x553b0e2?})\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/cmd/must-gather/query_infra_cmd.go:45 +0x28 fp=0x28002a593b80 sp=0x28002a593b58 pc=0x3938128\\ngithub.com/spf13/cobra.(*Command).execute(0x28002a53d508, {0x280029ab8120, 0xc, 0xc})\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1015 +0xb14 fp=0x28002a593d28 sp=0x28002a593b80 pc=0x212e614\\ngithub.com/spf13/cobra.(*Command).ExecuteC(0x280029ec9b08)\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1148 +0x465 fp=0x28002a593e18 sp=0x28002a593d28 pc=0x212efa5\\ngithub.com/spf13/cobra.(*Command).Execute(...)\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1071\\ngithub.com/spf13/cobra.(*Command).ExecuteContext(...)\\n\\t/go/pkg/mod/github.com/spf13/cobra@v1.10.2/command.go:1064\\nmain.main()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/main.go:129 +0x78a fp=0x28002a593f48 sp=0x28002a593e18 pc=0x47ddd6a\\nruntime.main()\\n\\t/usr/lib/golang/src/runtime/proc.go:290 +0x2d5 fp=0x28002a593fe0 sp=0x28002a593f48 pc=0x1f01975\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a593fe8 sp=0x28002a593fe0 pc=0x1f40801\\n\\ngoroutine 2 gp=0x280029a48d20 m=nil [force gc (idle)]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b08fa8 sp=0x280029b08f88 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.forcegchelper()\\n\\t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x280029b08fe0 sp=0x280029b08fa8 pc=0x1f01c93\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b08fe8 sp=0x280029b08fe0 pc=0x1f40801\\ncreated by runtime.init.7 in goroutine 1\\n\\t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\\n\\ngoroutine 3 gp=0x280029a492c0 m=nil [GC sweep wait]:\\nruntime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b09788 sp=0x280029b09768 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.bgsweep(0x280029b30000)\\n\\t/usr/lib/golang/src/runtime/mgcsweep.go:324 +0x151 fp=0x280029b097c8 sp=0x280029b09788 pc=0x1ee9f11\\nruntime.gcenable.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x280029b097e0 sp=0x280029b097c8 pc=0x1edadd7\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b097e8 sp=0x280029b097e0 pc=0x1f40801\\ncreated by runtime.gcenable in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\\n\\ngoroutine 4 gp=0x280029a494a0 m=nil [GC scavenge wait]:\\nruntime.gopark(0x10000?, 0x5981cd8?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b09f78 sp=0x280029b09f58 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.(*scavengerState).park(0x82979c0)\\n\\t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x280029b09fa8 sp=0x280029b09f78 pc=0x1ee77e9\\nruntime.bgscavenge(0x280029b30000)\\n\\t/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0x280029b09fc8 sp=0x280029b09fa8 pc=0x1ee7d99\\nruntime.gcenable.gowrap2()\\n\\t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x280029b09fe0 sp=0x280029b09fc8 pc=0x1edad97\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b09fe8 sp=0x280029b09fe0 pc=0x1f40801\\ncreated by runtime.gcenable in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\\n\\ngoroutine 5 gp=0x280029a49a40 m=nil [GOMAXPROCS updater (idle)]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0a788 sp=0x280029b0a768 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.updateMaxProcsGoroutine()\\n\\t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x280029b0a7e0 sp=0x280029b0a788 pc=0x1f10267\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0a7e8 sp=0x280029b0a7e0 pc=0x1f40801\\ncreated by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\\n\\t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\\n\\ngoroutine 6 gp=0x280029b481e0 m=nil [finalizer wait]:\\nruntime.gopark(0x1f11495?, 0x1c8?, 0x80?, 0x3e?, 0x280029b08601?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b08620 sp=0x280029b08600 pc=0x1f38d2e\\nruntime.runFinalizers()\\n\\t/usr/lib/golang/src/runtime/mfinal.go:210 +0x107 fp=0x280029b087e0 sp=0x280029b08620 pc=0x1ed9cc7\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b087e8 sp=0x280029b087e0 pc=0x1f40801\\ncreated by runtime.createfing in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mfinal.go:172 +0x3d\\n\\ngoroutine 7 gp=0x280029d3fc20 m=nil [cleanup wait]:\\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0af68 sp=0x280029b0af48 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.(*cleanupQueue).dequeue(0x82980e0)\\n\\t/usr/lib/golang/src/runtime/mcleanup.go:522 +0xd4 fp=0x280029b0afa0 sp=0x280029b0af68 pc=0x1ed6074\\nruntime.runCleanups()\\n\\t/usr/lib/golang/src/runtime/mcleanup.go:718 +0x45 fp=0x280029b0afe0 sp=0x280029b0afa0 pc=0x1ed66e5\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0afe8 sp=0x280029b0afe0 pc=0x1f40801\\ncreated by runtime.(*cleanupQueue).createGs in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mcleanup.go:672 +0xa5\\n\\ngoroutine 8 gp=0x280029ee6000 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125c86ae32d7?, 0x0?, 0x0?, 0x0?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0b740 sp=0x280029b0b720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x280029ad7ab0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x280029b0b7c8 sp=0x280029b0b740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b0b7e0 sp=0x280029b0b7c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0b7e8 sp=0x280029b0b7e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 18 gp=0x280029b843c0 m=nil [GC worker (idle)]:\\nruntime.gopark(0x125c662e5759?, 0x3?, 0xc2?, 0x10?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b04740 sp=0x280029b04720 pc=0x1f38d2e\\nruntime.gcBgMarkWorker(0x280029ad7ab0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x280029b047c8 sp=0x280029b04740 pc=0x1edd8ab\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b047e0 sp=0x280029b047c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b047e8 sp=0x280029b047e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 9 gp=0x280029ee61e0 m=nil [semacquire]:\\nruntime.gopark(0x6e2b22?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029b0bc96?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b0be38 sp=0x280029b0be18 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029b0bea0 sp=0x280029b0be38 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029b0bf40 sp=0x280029b0bea0 pc=0x1edbb12\\nruntime.gcBgMarkWorker(0x280029ad7ab0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029b0bfc8 sp=0x280029b0bf40 pc=0x1edda88\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b0bfe0 sp=0x280029b0bfc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b0bfe8 sp=0x280029b0bfe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 10 gp=0x280029ee63c0 m=nil [semacquire]:\\nruntime.gopark(0x1ec6f5e?, 0x1f3ec12?, 0x80?, 0x51?, 0x1ec70df?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eec638 sp=0x280029eec618 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eec6a0 sp=0x280029eec638 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eec740 sp=0x280029eec6a0 pc=0x1edbb12\\nruntime.gcBgMarkWorker(0x280029ad7ab0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eec7c8 sp=0x280029eec740 pc=0x1edda88\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eec7e0 sp=0x280029eec7c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eec7e8 sp=0x280029eec7e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 11 gp=0x280029ee65a0 m=nil [semacquire]:\\nruntime.gopark(0x54b522?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029eecc96?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eece38 sp=0x280029eece18 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eecea0 sp=0x280029eece38 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eecf40 sp=0x280029eecea0 pc=0x1edbb12\\nruntime.gcBgMarkWorker(0x280029ad7ab0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eecfc8 sp=0x280029eecf40 pc=0x1edda88\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eecfe0 sp=0x280029eecfc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eecfe8 sp=0x280029eecfe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 34 gp=0x280029f04000 m=nil [semacquire]:\\nruntime.gopark(0x2d7122?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029ee8496?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029ee8638 sp=0x280029ee8618 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029ee86a0 sp=0x280029ee8638 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029ee8740 sp=0x280029ee86a0 pc=0x1edbb12\\nruntime.gcBgMarkWorker(0x280029ad7ab0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029ee87c8 sp=0x280029ee8740 pc=0x1edda88\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029ee87e0 sp=0x280029ee87c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029ee87e8 sp=0x280029ee87e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 19 gp=0x280029b845a0 m=4 mp=0x280029b34008 [GC mark termination]:\\nruntime.systemstack_switch()\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x280029b04ea0 sp=0x280029b04e90 pc=0x1f3ebc8\\nruntime.forEachP(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:2112\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x280029b04f40 sp=0x280029b04ea0 pc=0x1edbc34\\nruntime.gcBgMarkWorker(0x280029ad7ab0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029b04fc8 sp=0x280029b04f40 pc=0x1edda88\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029b04fe0 sp=0x280029b04fc8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b04fe8 sp=0x280029b04fe0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 12 gp=0x280029ee6780 m=nil [semacquire]:\\nruntime.gopark(0x3f9e22?, 0x597ac58?, 0xc0?, 0xd4?, 0x280029eed496?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eed638 sp=0x280029eed618 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x280029eed6a0 sp=0x280029eed638 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x280029eed740 sp=0x280029eed6a0 pc=0x1edbb12\\nruntime.gcBgMarkWorker(0x280029ad7ab0)\\n\\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x280029eed7c8 sp=0x280029eed740 pc=0x1edda88\\nruntime.gcBgMarkStartWorkers.gowrap1()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x280029eed7e0 sp=0x280029eed7c8 pc=0x1edd797\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eed7e8 sp=0x280029eed7e0 pc=0x1f40801\\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\\n\\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\\n\\ngoroutine 35 gp=0x280029ee74a0 m=nil [select, locked to thread]:\\nruntime.gopark(0x280029eef7a8?, 0x2?, 0x1?, 0x74?, 0x280029eef794?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eef628 sp=0x280029eef608 pc=0x1f38d2e\\nruntime.selectgo(0x280029eef7a8, 0x280029eef790, 0x0?, 0x0, 0x0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x280029eef758 sp=0x280029eef628 pc=0x1f152a5\\nruntime.ensureSigM.func1()\\n\\t/usr/lib/golang/src/runtime/signal_unix.go:1091 +0x188 fp=0x280029eef7e0 sp=0x280029eef758 pc=0x1f328e8\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eef7e8 sp=0x280029eef7e0 pc=0x1f40801\\ncreated by runtime.ensureSigM in goroutine 1\\n\\t/usr/lib/golang/src/runtime/signal_unix.go:1074 +0xc5\\n\\ngoroutine 50 gp=0x280029f84960 m=9 mp=0x28002a000008 [syscall]:\\nruntime.notetsleepg(0x82bdea0, 0xffffffffffffffff)\\n\\t/usr/lib/golang/src/runtime/lock_futex.go:123 +0x29 fp=0x280029ee9fa0 sp=0x280029ee9f78 pc=0x1ece509\\nos/signal.signal_recv()\\n\\t/usr/lib/golang/src/runtime/sigqueue.go:152 +0x98 fp=0x280029ee9fc0 sp=0x280029ee9fa0 pc=0x1f3b298\\nos/signal.loop()\\n\\t/usr/lib/golang/src/os/signal/signal_unix.go:23 +0x13 fp=0x280029ee9fe0 sp=0x280029ee9fc0 pc=0x2030553\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029ee9fe8 sp=0x280029ee9fe0 pc=0x1f40801\\ncreated by os/signal.Notify.func1.1 in goroutine 1\\n\\t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\\n\\ngoroutine 51 gp=0x280029f84b40 m=nil [select]:\\nruntime.gopark(0x280029eea798?, 0x2?, 0x80?, 0xa3?, 0x280029eea78c?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029eea628 sp=0x280029eea608 pc=0x1f38d2e\\nruntime.selectgo(0x280029eea798, 0x280029eea788, 0x0?, 0x0, 0x0?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x280029eea758 sp=0x280029eea628 pc=0x1f152a5\\nos/signal.NotifyContext.func1()\\n\\t/usr/lib/golang/src/os/signal/signal.go:292 +0x7c fp=0x280029eea7e0 sp=0x280029eea758 pc=0x202ff9c\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029eea7e8 sp=0x280029eea7e0 pc=0x1f40801\\ncreated by os/signal.NotifyContext in goroutine 1\\n\\t/usr/lib/golang/src/os/signal/signal.go:291 +0x158\\n\\ngoroutine 137 gp=0x28002a60a000 m=nil [sync.WaitGroup.Wait]:\\nruntime.gopark(0x28002a603e00?, 0x1ed0765?, 0x80?, 0xea?, 0x7f5e54234a40?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a603da0 sp=0x28002a603d80 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x280029ae3b10, 0x0, 0x1, 0x0, 0x19)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a603e08 sp=0x28002a603da0 pc=0x1f162b2\\nsync.runtime_SemacquireWaitGroup(0x28002a83b590?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0x28002a603e40 sp=0x28002a603e08 pc=0x1f3a66e\\nsync.(*WaitGroup).Wait(0x280029ae3b08)\\n\\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0x28002a603e68 sp=0x28002a603e40 pc=0x1f4f445\\ngolang.org/x/sync/errgroup.(*Group).Wait(0x280029ae3b00)\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:56 +0x1e fp=0x28002a603e88 sp=0x28002a603e68 pc=0x362507e\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries(0x280029d80e80, {0x59f0b40, 0x280029beca00}, {0x28002a977b50, 0x1, 0x38d788c?}, 0x28002a830c40)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:83 +0x1ca fp=0x28002a603f00 sp=0x28002a603e88 pc=0x392976a\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:641 +0x6e fp=0x28002a603f78 sp=0x28002a603f00 pc=0x392808e\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a603fe0 sp=0x28002a603f78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a603fe8 sp=0x28002a603fe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 38 gp=0x28002a60b0e0 m=nil [IO wait]:\\nruntime.gopark(0x28002a7c22c0?, 0x280029aee7f8?, 0x40?, 0x77?, 0xb?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x280029b197b8 sp=0x280029b19798 pc=0x1f38d2e\\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x280029b197f0 sp=0x280029b197b8 pc=0x1ef9e57\\ninternal/poll.runtime_pollWait(0x7f5e554b6c00, 0x72)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x280029b19810 sp=0x280029b197f0 pc=0x1f375c5\\ninternal/poll.(*pollDesc).wait(0x280029bf6400?, 0x28002a90c000?, 0x0)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x280029b19838 sp=0x280029b19810 pc=0x1fbd167\\ninternal/poll.(*pollDesc).waitRead(...)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\\ninternal/poll.(*FD).Read(0x280029bf6400, {0x28002a90c000, 0x3500, 0x3500})\\n\\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x280029b198d0 sp=0x280029b19838 pc=0x1fbe4ce\\nnet.(*netFD).Read(0x280029bf6400, {0x28002a90c000?, 0x28002a90c000?, 0x5?})\\n\\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x280029b19918 sp=0x280029b198d0 pc=0x20798c5\\nnet.(*conn).Read(0x280029a46378, {0x28002a90c000?, 0x7f5e554a9828?, 0x7f5e9c0b1e00?})\\n\\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x280029b19960 sp=0x280029b19918 pc=0x208b7a5\\ncrypto/tls.(*atLeastReader).Read(0x28002aa26ba0, {0x28002a90c000?, 0x28002a0d8658?, 0x1efcc28?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x280029b199a8 sp=0x280029b19960 pc=0x2222ddb\\nbytes.(*Buffer).ReadFrom(0x28002a0d8628, {0x59a3680, 0x28002aa26ba0})\\n\\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x280029b19a00 sp=0x280029b199a8 pc=0x1fe9258\\ncrypto/tls.(*Conn).readFromUntil(0x28002a0d8388, {0x59a32c0, 0x280029a46378}, 0x222764c?)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x280029b19a38 sp=0x280029b19a00 pc=0x2222fbe\\ncrypto/tls.(*Conn).readRecordOrCCS(0x28002a0d8388, 0x0)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x280029b19c18 sp=0x280029b19a38 pc=0x222009b\\ncrypto/tls.(*Conn).readRecord(...)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:588\\ncrypto/tls.(*Conn).Read(0x28002a0d8388, {0x280029fed000, 0x1000, 0x22c34e0?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x280029b19c80 sp=0x280029b19c18 pc=0x22267c5\\nbufio.(*Reader).Read(0x28002a7b89c0, {0x28002a7c8584, 0x9, 0x22e100e?})\\n\\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0x280029b19cb8 sp=0x280029b19c80 pc=0x20a71f7\\nio.ReadAtLeast({0x59a1260, 0x28002a7b89c0}, {0x28002a7c8584, 0x9, 0x9}, 0x9)\\n\\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0x280029b19d00 sp=0x280029b19cb8 pc=0x1fb17ee\\nio.ReadFull(...)\\n\\t/usr/lib/golang/src/io/io.go:354\\nnet/http.http2readFrameHeader({0x28002a7c8584, 0x9, 0x280000000015?}, {0x59a1260?, 0x28002a7b89c0?})\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:1805 +0x65 fp=0x280029b19d50 sp=0x280029b19d00 pc=0x22b9e25\\nnet/http.(*http2Framer).ReadFrameHeader(0x28002a7c8540)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:2071 +0x6b fp=0x280029b19dd8 sp=0x280029b19d50 pc=0x22ba9eb\\nnet/http.(*http2Framer).ReadFrame(0x28002a7c8540)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:2130 +0x18 fp=0x280029b19e18 sp=0x280029b19dd8 pc=0x22bb1b8\\nnet/http.(*http2clientConnReadLoop).run(0x280029b19fa8)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9550 +0xca fp=0x280029b19f60 sp=0x280029b19e18 pc=0x22de14a\\nnet/http.(*http2ClientConn).readLoop(0x280029bc01c0)\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:9419 +0x52 fp=0x280029b19fc8 sp=0x280029b19f60 pc=0x22dd532\\nnet/http.(*http2Transport).newClientConn.gowrap1()\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0x17 fp=0x280029b19fe0 sp=0x280029b19fc8 pc=0x22d7057\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x280029b19fe8 sp=0x280029b19fe0 pc=0x1f40801\\ncreated by net/http.(*http2Transport).newClientConn in goroutine 37\\n\\t/usr/lib/golang/src/net/http/h2_bundle.go:8171 +0xda5\\n\\ngoroutine 69 gp=0x28002a60b4a0 m=nil [IO wait]:\\nruntime.gopark(0x28002a50c560?, 0x1edb176?, 0x80?, 0xed?, 0xb?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a606808 sp=0x28002a6067e8 pc=0x1f38d2e\\nruntime.netpollblock(0x1f601d8?, 0x1ec4026?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0x28002a606840 sp=0x28002a606808 pc=0x1ef9e57\\ninternal/poll.runtime_pollWait(0x7f5e554b6a00, 0x72)\\n\\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0x28002a606860 sp=0x28002a606840 pc=0x1f375c5\\ninternal/poll.(*pollDesc).wait(0x280029bf6c80?, 0x280029a5a000?, 0x0)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0x28002a606888 sp=0x28002a606860 pc=0x1fbd167\\ninternal/poll.(*pollDesc).waitRead(...)\\n\\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\\ninternal/poll.(*FD).Read(0x280029bf6c80, {0x280029a5a000, 0x2500, 0x2500})\\n\\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x2ae fp=0x28002a606920 sp=0x28002a606888 pc=0x1fbe4ce\\nnet.(*netFD).Read(0x280029bf6c80, {0x280029a5a000?, 0x28002a60690a?, 0x1edb176?})\\n\\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0x28002a606968 sp=0x28002a606920 pc=0x20798c5\\nnet.(*conn).Read(0x280029a463c0, {0x280029a5a000?, 0x7f5e5423eae0?, 0x7f5e9c09dde0?})\\n\\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0x28002a6069b0 sp=0x28002a606968 pc=0x208b7a5\\ncrypto/tls.(*atLeastReader).Read(0x28002a628318, {0x280029a5a000?, 0x28002a606a98?, 0x1efcc28?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:815 +0x3b fp=0x28002a6069f8 sp=0x28002a6069b0 pc=0x2222ddb\\nbytes.(*Buffer).ReadFrom(0x280029ba9ea8, {0x59a3680, 0x28002a628318})\\n\\t/usr/lib/golang/src/bytes/buffer.go:229 +0x98 fp=0x28002a606a50 sp=0x28002a6069f8 pc=0x1fe9258\\ncrypto/tls.(*Conn).readFromUntil(0x280029ba9c08, {0x59a32c0, 0x280029a463c0}, 0x222764c?)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:837 +0xde fp=0x28002a606a88 sp=0x28002a606a50 pc=0x2222fbe\\ncrypto/tls.(*Conn).readRecordOrCCS(0x280029ba9c08, 0x0)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:626 +0x3db fp=0x28002a606c68 sp=0x28002a606a88 pc=0x222009b\\ncrypto/tls.(*Conn).readRecord(...)\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:588\\ncrypto/tls.(*Conn).Read(0x280029ba9c08, {0x28002a24a000, 0x1000, 0x4e55540?})\\n\\t/usr/lib/golang/src/crypto/tls/conn.go:1393 +0x145 fp=0x28002a606cd0 sp=0x28002a606c68 pc=0x22267c5\\nnet/http.(*persistConn).Read(0x28002a0ad400, {0x28002a24a000?, 0x59a0ea0?, 0x81260e0?})\\n\\t/usr/lib/golang/src/net/http/transport.go:2174 +0x47 fp=0x28002a606d30 sp=0x28002a606cd0 pc=0x2319d07\\nbufio.(*Reader).fill(0x28002a8130e0)\\n\\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0x28002a606d68 sp=0x28002a606d30 pc=0x20a6c63\\nbufio.(*Reader).Peek(0x28002a8130e0, 0x1)\\n\\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0x28002a606d80 sp=0x28002a606d68 pc=0x20a6df3\\nnet/http.(*persistConn).readLoop(0x28002a0ad400)\\n\\t/usr/lib/golang/src/net/http/transport.go:2330 +0x172 fp=0x28002a606fc8 sp=0x28002a606d80 pc=0x231a812\\nnet/http.(*Transport).dialConn.gowrap2()\\n\\t/usr/lib/golang/src/net/http/transport.go:1994 +0x17 fp=0x28002a606fe0 sp=0x28002a606fc8 pc=0x2319197\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a606fe8 sp=0x28002a606fe0 pc=0x1f40801\\ncreated by net/http.(*Transport).dialConn in goroutine 55\\n\\t/usr/lib/golang/src/net/http/transport.go:1994 +0x16ac\\n\\ngoroutine 70 gp=0x28002a60bc20 m=nil [select]:\\nruntime.gopark(0x28002a613f48?, 0x2?, 0x0?, 0xb6?, 0x28002a613ef4?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a708d88 sp=0x28002a708d68 pc=0x1f38d2e\\nruntime.selectgo(0x28002a708f48, 0x28002a613ef0, 0x280029c93a80?, 0x0, 0x28002a4f7c80?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a708eb8 sp=0x28002a708d88 pc=0x1f152a5\\nnet/http.(*persistConn).writeLoop(0x28002a0ad400)\\n\\t/usr/lib/golang/src/net/http/transport.go:2652 +0xe6 fp=0x28002a708fc8 sp=0x28002a708eb8 pc=0x231c426\\nnet/http.(*Transport).dialConn.gowrap3()\\n\\t/usr/lib/golang/src/net/http/transport.go:1995 +0x17 fp=0x28002a708fe0 sp=0x28002a708fc8 pc=0x2319157\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a708fe8 sp=0x28002a708fe0 pc=0x1f40801\\ncreated by net/http.(*Transport).dialConn in goroutine 55\\n\\t/usr/lib/golang/src/net/http/transport.go:1995 +0x16fe\\n\\ngoroutine 138 gp=0x28002a84a780 m=nil [semacquire]:\\nruntime.gopark(0x1?, 0x1f78fa5?, 0xe0?, 0x75?, 0x2?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a2ca910 sp=0x28002a2ca8f0 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a2ca978 sp=0x28002a2ca910 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x28002a2caa18 sp=0x28002a2ca978 pc=0x1edbb12\\nruntime.gcAssistAlloc(0x28002a84a780)\\n\\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x28002a2caaa8 sp=0x28002a2caa18 pc=0x1ee06b6\\nruntime.deductAssistCredit(0x28002a2cac70?)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x28002a2caac8 sp=0x28002a2caaa8 pc=0x1ed10a6\\nruntime.mallocgc(0x10, 0x4af39e0, 0x1)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x28002a2caaf8 sp=0x28002a2caac8 pc=0x1f35e07\\nruntime.convTstring({0x28002a6a7728, 0x14})\\n\\t/usr/lib/golang/src/runtime/iface.go:431 +0x45 fp=0x28002a2cab20 sp=0x28002a2caaf8 pc=0x1f35865\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.cliOutputFunc({0x59f0b40, 0x280029beca00}, 0x28002a830cb0, 0x280029f636e0)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:297 +0x394 fp=0x28002a2caf48 sp=0x28002a2cab20 pc=0x3920674\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func2()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:645 +0x2c fp=0x28002a2caf78 sp=0x28002a2caf48 pc=0x392800c\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a2cafe0 sp=0x28002a2caf78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a2cafe8 sp=0x28002a2cafe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 139 gp=0x28002a84a960 m=nil [select]:\\nruntime.gopark(0x28002a4bdd00?, 0x2?, 0x1?, 0xc9?, 0x28002a4bdc74?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a4bdac8 sp=0x28002a4bdaa8 pc=0x1f38d2e\\nruntime.selectgo(0x28002a4bdd00, 0x28002a4bdc70, 0x28002a3e8a50?, 0x0, 0x280029f827c8?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a4bdbf8 sp=0x28002a4bdac8 pc=0x1f152a5\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).convertRows(0x49424d554554534c?, {0x59f0b40, 0x280029beca00}, 0x28002a830c40, 0x28002a830cb0)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:673 +0x24e fp=0x28002a4bdf10 sp=0x28002a4bdbf8 pc=0x392836e\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*Gatherer).queryAndWriteToFile.func3()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/gather.go:650 +0x5d fp=0x28002a4bdf78 sp=0x28002a4bdf10 pc=0x3927f3d\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a4bdfe0 sp=0x28002a4bdf78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a4bdfe8 sp=0x28002a4bdfe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 140 gp=0x28002a84ab40 m=nil [semacquire]:\\nruntime.gopark(0x1f6aa32?, 0x28002a34b418?, 0xc5?, 0xbd?, 0x0?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a34b3c0 sp=0x28002a34b3a0 pc=0x1f38d2e\\nruntime.goparkunlock(...)\\n\\t/usr/lib/golang/src/runtime/proc.go:468\\nruntime.semacquire1(0x8299ce4, 0x0, 0x0, 0x0, 0xd)\\n\\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x28002a34b428 sp=0x28002a34b3c0 pc=0x1f162b2\\nruntime.semacquire(...)\\n\\t/usr/lib/golang/src/runtime/sema.go:143\\nruntime.gcMarkDone()\\n\\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x28002a34b4c8 sp=0x28002a34b428 pc=0x1edbb12\\nruntime.gcAssistAlloc(0x28002a84ab40)\\n\\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x28002a34b558 sp=0x28002a34b4c8 pc=0x1ee06b6\\nruntime.deductAssistCredit(0x20?)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x28002a34b578 sp=0x28002a34b558 pc=0x1ed10a6\\nruntime.mallocgc(0x1000, 0x4af32a0, 0x1)\\n\\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x28002a34b5a8 sp=0x28002a34b578 pc=0x1f35e07\\nruntime.makeslice(0x28002a34b668?, 0x1f8?, 0x1ff?)\\n\\t/usr/lib/golang/src/runtime/slice.go:117 +0x49 fp=0x28002a34b5d0 sp=0x28002a34b5a8 pc=0x1f3b489\\nbufio.NewWriterSize(...)\\n\\t/usr/lib/golang/src/bufio/bufio.go:599\\nbufio.NewWriter(...)\\n\\t/usr/lib/golang/src/bufio/bufio.go:608\\nencoding/csv.NewWriter(...)\\n\\t/usr/lib/golang/src/encoding/csv/writer.go:42\\ngithub.com/Azure/azure-kusto-go/azkustodata/query.(*row).String(0x812ab90?)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/row_impl.go:100 +0x18e fp=0x28002a34b6e8 sp=0x28002a34b5d0 pc=0x38cf5ee\\nfmt.(*pp).handleMethods(0x28002a24c0d0, 0x2a34b8f0?)\\n\\t/usr/lib/golang/src/fmt/print.go:674 +0x2f4 fp=0x28002a34b8a8 sp=0x28002a34b6e8 pc=0x1fdd1d4\\nfmt.(*pp).printArg(0x28002a24c0d0, {0x549a3a0, 0x28002a7c7c80}, 0x76)\\n\\t/usr/lib/golang/src/fmt/print.go:757 +0x42e fp=0x28002a34b938 sp=0x28002a34b8a8 pc=0x1fdd9ee\\nfmt.(*pp).doPrintf(0x28002a24c0d0, {0x553a400, 0x2}, {0x28002a34bd08, 0x1, 0x1})\\n\\t/usr/lib/golang/src/fmt/print.go:1075 +0x3ec fp=0x28002a34ba20 sp=0x28002a34b938 pc=0x1fe04ec\\nfmt.Sprintf({0x553a400, 0x2}, {0x28002a34bd08, 0x1, 0x1})\\n\\t/usr/lib/golang/src/fmt/print.go:239 +0x53 fp=0x28002a34ba78 sp=0x28002a34ba20 pc=0x1fda6b3\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0x280029e454e0, {0x59f0b40, 0x280029beca50}, {0x5a00300, 0x280029ae2040}, 0x28002a830c40)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:160 +0x8d8 fp=0x28002a34bd60 sp=0x28002a34ba78 pc=0x391bad8\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ExecutePreconfiguredQuery(0x2d20414320746f6f?, {0x59f0b40?, 0x280029beca50?}, {0x5a00300?, 0x280029ae2040?}, 0x43437a4b4349494d?)\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:111 +0xb6 fp=0x28002a34be88 sp=0x28002a34bd60 pc=0x3929c16\\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\\n\\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x6a fp=0x28002a34bf78 sp=0x28002a34be88 pc=0x392984a\\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:93 +0x50 fp=0x28002a34bfe0 sp=0x28002a34bf78 pc=0x36251d0\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a34bfe8 sp=0x28002a34bfe0 pc=0x1f40801\\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 137\\n\\t/go/pkg/mod/golang.org/x/sync@v0.23.0/errgroup/errgroup.go:78 +0x95\\n\\ngoroutine 142 gp=0x28002a84af00 m=nil [select]:\\nruntime.gopark(0x28002a333de0?, 0x2?, 0x8?, 0xb8?, 0x28002a333dbc?)\\n\\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x28002a333c58 sp=0x28002a333c38 pc=0x1f38d2e\\nruntime.selectgo(0x28002a333de0, 0x28002a333db8, 0x280029afc008?, 0x1, 0x1f3665f?, 0x1)\\n\\t/usr/lib/golang/src/runtime/select.go:351 +0xaa5 fp=0x28002a333d88 sp=0x28002a333c58 pc=0x1f152a5\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.(*iterativeDataset).sendTable(0x280029becf00, {0x5a19be0, 0x28002ab876c0})\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:378 +0xf3 fp=0x28002a333e10 sp=0x28002a333d88 pc=0x38d8d33\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.handleDataTable(0x280029becf00, 0x28002a3b2b40)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:324 +0x2c6 fp=0x28002a333ec8 sp=0x28002a333e10 pc=0x38d8686\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.readDataSet(0x280029becf00)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:167 +0x148 fp=0x28002a333f30 sp=0x28002a333ec8 pc=0x38d7b48\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.parseRoutine(0x280029becf00, 0x28002a977da0)\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:117 +0x2c fp=0x28002a333fc0 sp=0x28002a333f30 pc=0x38d788c\\ngithub.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset.gowrap1()\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1b fp=0x28002a333fe0 sp=0x28002a333fc0 pc=0x38d751b\\nruntime.goexit({})\\n\\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x28002a333fe8 sp=0x28002a333fe0 pc=0x1f40801\\ncreated by github.com/Azure/azure-kusto-go/azkustodata/query/v2.NewIterativeDataset in goroutine 140\\n\\t/go/pkg/mod/github.com/!azure/azure-kusto-go/azkustodata@v1.2.1/query/v2/iterative_dataset.go:73 +0x1fc\\n\\nrax    0x0\\nrbx    0x7f5e554a8640\\nrcx    0x7f5e9c16994c\\nrdx    0x6\\nrdi    0x10a66\\nrsi    0x10a67\\nrbp    0x10a67\\nrsp    0x7f5e554a7740\\nr8     0x7f5e554a7810\\nr9     0x7f5e9c28f4e0\\nr10    0x8\\nr11    0x246\\nr12    0x6\\nr13    0x7f5e554a7b10\\nr14    0x280029a48b40\\nr15    0xffffffffffffffff\\nrip    0x7f5e9c16994c\\nrflags 0x246\\ncs     0x33\\nfs     0x0\\ngs     0x0\\n\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:123]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc8e80bea4a0\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[17:06:46.277]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575\"\n    }�[0m\n    �[37m[17:06:50.310]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:06:48.7179236Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\\\", ActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", SubActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \\u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[17:06:50.310]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 78/289 failed (KQL: let _startTime = datetime(2026-10-14T23:31:46.2770422Z);\\nlet _endTime = datetime(2026-10-14T23:36:46.2770421Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[17:06:50.376]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/cosmosContent\"\n    }�[0m\n    �[37m[17:06:50.376]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[17:06:50.697]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 1106,\n      \"files\": 1\n    }�[0m\n    �[37m[17:06:50.697]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/cosmosContent\"\n    }�[0m\n    �[37m[17:06:50.706]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[17:07:03.236]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n      \"commits\": 613\n    }�[0m\n    �[37m[17:07:03.237]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect\"\n    }�[0m\n    �[37m[17:07:03.356]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n      ]\n    }�[0m\n    �[37m[17:07:03.470]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j0325120-mgmt-2\",\n        \"ci01-j0325120-svc\"\n      ]\n    }�[0m\n    �[37m[17:07:03.503]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaces\": [\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n      ]\n    }�[0m\n    �[37m[17:07:03.503]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaceCount\": 2\n    }�[0m\n    �[37m[17:07:03.503]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n    }�[0m\n    �[37m[17:07:03.650]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n    }�[0m\n    �[37m[17:07:04.108]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:06:48.7179236Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\\\", ActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", SubActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \\u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\\\"}, IsPermanent:false})\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575\"\n    }�[0m\n    �[37m[17:07:04.108]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:06:48.7179236Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\\\", ActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", SubActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \\u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\\\"}, IsPermanent:false})\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[17:07:04.396]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816\"\n    }�[0m\n    �[37m[17:07:13.034]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:11.3621455Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\\\", ActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", SubActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \\u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[17:07:13.034]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 83/289 failed (KQL: let _startTime = datetime(2026-10-14T23:57:04.3961354Z);\\nlet _endTime = datetime(2026-10-15T00:02:04.3961353Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[17:07:13.071]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/cosmosContent\"\n    }�[0m\n    �[37m[17:07:13.072]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[17:07:13.393]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 1106,\n      \"files\": 1\n    }�[0m\n    �[37m[17:07:13.393]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/cosmosContent\"\n    }�[0m\n    �[37m[17:07:13.403]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[17:07:25.804]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n      \"commits\": 613\n    }�[0m\n    �[37m[17:07:25.805]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect\"\n    }�[0m\n    �[37m[17:07:25.912]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n      ]\n    }�[0m\n    �[37m[17:07:26.021]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j0325120-mgmt-2\",\n        \"ci01-j0325120-svc\"\n      ]\n    }�[0m\n    �[37m[17:07:26.141]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaces\": [\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n      ]\n    }�[0m\n    �[37m[17:07:26.141]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaceCount\": 2\n    }�[0m\n    �[37m[17:07:26.141]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n    }�[0m\n    �[37m[17:07:26.291]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n    }�[0m\n    �[37m[17:07:26.780]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:11.3621455Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\\\", ActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", SubActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \\u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\\\"}, IsPermanent:false})\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816\"\n    }�[0m\n    �[37m[17:07:26.780]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:11.3621455Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\\\", ActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", SubActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \\u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\\\"}, IsPermanent:false})\"\n    }�[0m\n    \n      query-skip-kubernetes-events: command failed: exit status 1\n    output: �[37m[17:07:27.073]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188\"\n    }�[0m\n    �[37m[17:07:30.812]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:29.3103489Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\\\", ActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", SubActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \\u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[17:07:30.812]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 76/289 failed (KQL: let _startTime = datetime(2026-10-14T23:22:27.0728956Z);\\nlet _endTime = datetime(2026-10-14T23:27:27.0728955Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-14T23:22:27.0728956Z);\\nlet _endTime = datetime(2026-10-14T23:27:27.0728955Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[17:07:30.813]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/cosmosContent\"\n    }�[0m\n    �[37m[17:07:30.813]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[17:07:31.138]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 1106,\n      \"files\": 1\n    }�[0m\n    �[37m[17:07:31.138]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/cosmosContent\"\n    }�[0m\n    �[37m[17:07:31.148]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[17:07:42.926]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect\"\n    }�[0m\n    �[37m[17:07:43.042]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n      ]\n    }�[0m\n    �[37m[17:07:43.166]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j0325120-mgmt-2\",\n        \"ci01-j0325120-svc\"\n      ]\n    }�[0m\n    �[37m[17:07:43.202]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaces\": [\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n      ]\n    }�[0m\n    �[37m[17:07:43.202]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaceCount\": 2\n    }�[0m\n    �[37m[17:07:43.202]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n    }�[0m\n    �[37m[17:07:43.435]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n    }�[0m\n    �[37m[17:07:43.942]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:29.3103489Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\\\", ActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", SubActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \\u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188\"\n    }�[0m\n    �[37m[17:07:43.942]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:29.3103489Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\\\", ActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", SubActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \\u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[17:07:44.226]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200\"\n    }�[0m\n    �[37m[17:07:47.896]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:46.3200405Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\\\", ActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", SubActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \\u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[17:07:47.896]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 76/289 failed (KQL: let _startTime = datetime(2026-10-14T23:22:44.2253704Z);\\nlet _endTime = datetime(2026-10-14T23:27:44.2253703Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-14T23:22:44.2253704Z);\\nlet _endTime = datetime(2026-10-14T23:27:44.2253703Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[17:07:47.968]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/cosmosContent\"\n    }�[0m\n    �[37m[17:07:47.968]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[17:07:48.283]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 1106,\n      \"files\": 1\n    }�[0m\n    �[37m[17:07:48.283]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/cosmosContent\"\n    }�[0m\n    �[37m[17:07:48.293]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[17:08:00.494]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n      \"commits\": 613\n    }�[0m\n    �[37m[17:08:00.495]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect\"\n    }�[0m\n    �[37m[17:08:00.610]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n      ]\n    }�[0m\n    �[37m[17:08:00.715]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j0325120-mgmt-2\",\n        \"ci01-j0325120-svc\"\n      ]\n    }�[0m\n    �[37m[17:08:00.747]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaces\": [\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n      ]\n    }�[0m\n    �[37m[17:08:00.747]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaceCount\": 2\n    }�[0m\n    �[37m[17:08:00.747]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n    }�[0m\n    �[37m[17:08:01.068]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n    }�[0m\n    �[37m[17:08:01.547]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:46.3200405Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\\\", ActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", SubActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \\u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\\\"}, IsPermanent:false})\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200\"\n    }�[0m\n    �[37m[17:08:01.548]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:46.3200405Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\\\", ActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", SubActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \\u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\\\"}, IsPermanent:false})\"\n    }�[0m\n    \n      query-skip-hcp-and-events: command failed: exit status 1\n    output: �[37m[17:08:01.848]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533\"\n    }�[0m\n    �[37m[17:08:05.427]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:08:03.7881995Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\\\", ActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", SubActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \\u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[17:08:05.428]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 85/289 failed (KQL: let _startTime = datetime(2026-10-15T00:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T00:13:01.8473530Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-mgmt-2', 'ci01-j0325120-svc', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-15T00:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T00:13:01.8473530Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-mgmt-2', 'ci01-j0325120-svc', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[17:08:05.428]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/cosmosContent\"\n    }�[0m\n    �[37m[17:08:05.428]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[17:08:05.743]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 1106,\n      \"files\": 1\n    }�[0m\n    �[37m[17:08:05.743]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/cosmosContent\"\n    }�[0m\n    �[37m[17:08:05.752]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[17:08:15.101]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect\"\n    }�[0m\n    �[37m[17:08:15.221]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n      ]\n    }�[0m\n    �[37m[17:08:15.326]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j0325120-mgmt-2\",\n        \"ci01-j0325120-svc\"\n      ]\n    }�[0m\n    �[37m[17:08:15.357]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaces\": [\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n      ]\n    }�[0m\n    �[37m[17:08:15.357]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaceCount\": 2\n    }�[0m\n    �[37m[17:08:15.357]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n    }�[0m\n    �[37m[17:08:15.495]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n    }�[0m\n    �[37m[17:08:15.976]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:08:03.7881995Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\\\", ActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", SubActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \\u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533\"\n    }�[0m\n    �[37m[17:08:15.977]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:08:03.7881995Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\\\", ActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", SubActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \\u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n    }�[0m\n    \n    {\n        s: \"must-gather CLI test failures:\\n  query-basic: command failed: exit status 1\\noutput: \\x1b[37m[17:06:46.277]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-824225575\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:06:50.310]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\\\\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:06:48.7179236Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\\\\\\\", ActivityId:\\\\\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\\\\\", SubActivityId:\\\\\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \\\\u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:06:50.310]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 78/289 failed (KQL: let _startTime = datetime(2026-10-14T23:31:46.2770422Z);\\\\nlet _endTime = datetime(2026-10-14T23:36:46.2770421Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:06:50.376]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-824225575/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-824225575/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:06:50.376]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-824225575/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:06:50.697]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 1106,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[17:06:50.697]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-824225575/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:06:50.706]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[17:07:03.236]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n  \\\"commits\\\": 613\\n}\\x1b[0m\\n\\x1b[37m[17:07:03.237]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:03.356]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:07:03.470]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j0325120-mgmt-2\\\",\\n    \\\"ci01-j0325120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:07:03.503]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaces\\\": [\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:07:03.503]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaceCount\\\": 2\\n}\\x1b[0m\\n\\x1b[37m[17:07:03.503]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:03.650]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:04.108]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\\\\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:06:48.7179236Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\\\\\\\", ActivityId:\\\\\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\\\\\", SubActivityId:\\\\\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \\\\u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-824225575\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:04.108]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\\\\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:06:48.7179236Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\\\\\\\", ActivityId:\\\\\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\\\\\", SubActivityId:\\\\\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \\\\u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n  query-skip-hcp-logs: command failed: exit status 1\\noutput: \\x1b[37m[17:07:04.396]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:13.034]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\\\\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:07:11.3621455Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\\\\\\\", ActivityId:\\\\\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\\\\\", SubActivityId:\\\\\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \\\\u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:13.034]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 83/289 failed (KQL: let _startTime = datetime(2026-10-14T23:57:04.3961354Z);\\\\nlet _endTime = datetime(2026-10-15T00:02:04.3961353Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:13.071]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:13.072]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:13.393]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 1106,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[17:07:13.393]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:13.403]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[17:07:25.804]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n  \\\"commits\\\": 613\\n}\\x1b[0m\\n\\x1b[37m[17:07:25.805]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:25.912]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:07:26.021]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j0325120-mgmt-2\\\",\\n    \\\"ci01-j0325120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:07:26.141]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaces\\\": [\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:07:26.141]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaceCount\\\": 2\\n}\\x1b[0m\\n\\x1b[37m[17:07:26.141]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:26.291]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:26.780]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\\\\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:07:11.3621455Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\\\\\\\", ActivityId:\\\\\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\\\\\", SubActivityId:\\\\\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \\\\u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:26.780]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\\\\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:07:11.3621455Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\\\\\\\", ActivityId:\\\\\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\\\\\", SubActivityId:\\\\\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \\\\u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n  query-skip-kubernetes-events: command failed: exit status 1\\noutput: \\x1b[37m[17:07:27.073]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:30.812]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\\\\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:07:29.3103489Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\\\\\\\", ActivityId:\\\\\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\\\\\", SubActivityId:\\\\\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \\\\u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:30.812]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 76/289 failed (KQL: let _startTime = datetime(2026-10-14T23:22:27.0728956Z);\\\\nlet _endTime = datetime(2026-10-14T23:27:27.0728955Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-14T23:22:27.0728956Z);\\\\nlet _endTime = datetime(2026-10-14T23:27:27.0728955Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:30.813]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:30.813]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:31.138]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 1106,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[17:07:31.138]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:31.148]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[17:07:42.926]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:43.042]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:07:43.166]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j0325120-mgmt-2\\\",\\n    \\\"ci01-j0325120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:07:43.202]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaces\\\": [\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:07:43.202]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaceCount\\\": 2\\n}\\x1b[0m\\n\\x1b[37m[17:07:43.202]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:43.435]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:43.942]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\\\\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:07:29.3103489Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\\\\\\\", ActivityId:\\\\\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\\\\\", SubActivityId:\\\\\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \\\\u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:43.942]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\\\\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:07:29.3103489Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\\\\\\\", ActivityId:\\\\\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\\\\\", SubActivityId:\\\\\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \\\\u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\"\\n}\\x1b[0m\\n\\n  query-collect-systemd-logs: command failed: exit status 1\\noutput: \\x1b[37m[17:07:44.226]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:47.896]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\\\\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:07:46.3200405Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\\\\\\\", ActivityId:\\\\\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\\\\\", SubActivityId:\\\\\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \\\\u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:47.896]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 76/289 failed (KQL: let _startTime = datetime(2026-10-14T23:22:44.2253704Z);\\\\nlet _endTime = datetime(2026-10-14T23:27:44.2253703Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-14T23:22:44.2253704Z);\\\\nlet _endTime = datetime(2026-10-14T23:27:44.2253703Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:47.968]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:47.968]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:48.283]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 1106,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[17:07:48.283]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:48.293]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[17:08:00.494]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n  \\\"commits\\\": 613\\n}\\x1b[0m\\n\\x1b[37m[17:08:00.495]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:00.610]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:08:00.715]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j0325120-mgmt-2\\\",\\n    \\\"ci01-j0325120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:08:00.747]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaces\\\": [\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:08:00.747]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaceCount\\\": 2\\n}\\x1b[0m\\n\\x1b[37m[17:08:00.747]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:01.068]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:01.547]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\\\\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:07:46.3200405Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\\\\\\\", ActivityId:\\\\\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\\\\\", SubActivityId:\\\\\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \\\\u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:01.548]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\\\\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:07:46.3200405Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\\\\\\\", ActivityId:\\\\\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\\\\\", SubActivityId:\\\\\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \\\\u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n  query-skip-hcp-and-events: command failed: exit status 1\\noutput: \\x1b[37m[17:08:01.848]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:05.427]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\\\\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:08:03.7881995Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\\\\\\\", ActivityId:\\\\\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\\\\\", SubActivityId:\\\\\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \\\\u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:05.428]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 85/289 failed (KQL: let _startTime = datetime(2026-10-15T00:08:01.8473531Z);\\\\nlet _endTime = datetime(2026-10-15T00:13:01.8473530Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j0325120-mgmt-2', 'ci01-j0325120-svc', 'ci01-j0325120-mgmt-2')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-15T00:08:01.8473531Z);\\\\nlet _endTime = datetime(2026-10-15T00:13:01.8473530Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j0325120-mgmt-2', 'ci01-j0325120-svc', 'ci01-j0325120-mgmt-2')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:05.428]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:05.428]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:05.743]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 1106,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[17:08:05.743]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:05.752]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[17:08:15.101]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:15.221]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:08:15.326]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j0325120-mgmt-2\\\",\\n    \\\"ci01-j0325120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:08:15.357]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaces\\\": [\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:08:15.357]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaceCount\\\": 2\\n}\\x1b[0m\\n\\x1b[37m[17:08:15.357]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:15.495]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:15.976]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\\\\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:08:03.7881995Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\\\\\\\", ActivityId:\\\\\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\\\\\", SubActivityId:\\\\\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \\\\u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:15.977]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\\\\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:08:03.7881995Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\\\\\\\", ActivityId:\\\\\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\\\\\", SubActivityId:\\\\\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \\\\u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\"\\n}\\x1b[0m\\n\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:123]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc8e80bea4a0\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[17:06:46.277]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575\"\n    }�[0m\n    �[37m[17:06:50.310]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:06:48.7179236Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\\\", ActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", SubActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \\u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[17:06:50.310]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 78/289 failed (KQL: let _startTime = datetime(2026-10-14T23:31:46.2770422Z);\\nlet _endTime = datetime(2026-10-14T23:36:46.2770421Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[17:06:50.376]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/cosmosContent\"\n    }�[0m\n    �[37m[17:06:50.376]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[17:06:50.697]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 1106,\n      \"files\": 1\n    }�[0m\n    �[37m[17:06:50.697]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/cosmosContent\"\n    }�[0m\n    �[37m[17:06:50.706]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[17:07:03.236]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n      \"commits\": 613\n    }�[0m\n    �[37m[17:07:03.237]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect\"\n    }�[0m\n    �[37m[17:07:03.356]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n      ]\n    }�[0m\n    �[37m[17:07:03.470]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j0325120-mgmt-2\",\n        \"ci01-j0325120-svc\"\n      ]\n    }�[0m\n    �[37m[17:07:03.503]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaces\": [\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n      ]\n    }�[0m\n    �[37m[17:07:03.503]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaceCount\": 2\n    }�[0m\n    �[37m[17:07:03.503]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n    }�[0m\n    �[37m[17:07:03.650]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n    }�[0m\n    �[37m[17:07:04.108]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:06:48.7179236Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\\\", ActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", SubActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \\u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\\\"}, IsPermanent:false})\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-824225575\"\n    }�[0m\n    �[37m[17:07:04.108]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:06:48.7179236Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\\\", ActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", SubActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \\u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\\\"}, IsPermanent:false})\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[17:07:04.396]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816\"\n    }�[0m\n    �[37m[17:07:13.034]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:11.3621455Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\\\", ActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", SubActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \\u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[17:07:13.034]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 83/289 failed (KQL: let _startTime = datetime(2026-10-14T23:57:04.3961354Z);\\nlet _endTime = datetime(2026-10-15T00:02:04.3961353Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[17:07:13.071]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/cosmosContent\"\n    }�[0m\n    �[37m[17:07:13.072]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[17:07:13.393]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 1106,\n      \"files\": 1\n    }�[0m\n    �[37m[17:07:13.393]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/cosmosContent\"\n    }�[0m\n    �[37m[17:07:13.403]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[17:07:25.804]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n      \"commits\": 613\n    }�[0m\n    �[37m[17:07:25.805]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect\"\n    }�[0m\n    �[37m[17:07:25.912]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n      ]\n    }�[0m\n    �[37m[17:07:26.021]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j0325120-mgmt-2\",\n        \"ci01-j0325120-svc\"\n      ]\n    }�[0m\n    �[37m[17:07:26.141]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaces\": [\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n      ]\n    }�[0m\n    �[37m[17:07:26.141]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaceCount\": 2\n    }�[0m\n    �[37m[17:07:26.141]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n    }�[0m\n    �[37m[17:07:26.291]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n    }�[0m\n    �[37m[17:07:26.780]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:11.3621455Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\\\", ActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", SubActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \\u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\\\"}, IsPermanent:false})\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816\"\n    }�[0m\n    �[37m[17:07:26.780]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:11.3621455Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\\\", ActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", SubActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \\u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\\\"}, IsPermanent:false})\"\n    }�[0m\n    \n      query-skip-kubernetes-events: command failed: exit status 1\n    output: �[37m[17:07:27.073]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188\"\n    }�[0m\n    �[37m[17:07:30.812]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:29.3103489Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\\\", ActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", SubActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \\u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[17:07:30.812]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 76/289 failed (KQL: let _startTime = datetime(2026-10-14T23:22:27.0728956Z);\\nlet _endTime = datetime(2026-10-14T23:27:27.0728955Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-14T23:22:27.0728956Z);\\nlet _endTime = datetime(2026-10-14T23:27:27.0728955Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[17:07:30.813]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/cosmosContent\"\n    }�[0m\n    �[37m[17:07:30.813]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[17:07:31.138]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 1106,\n      \"files\": 1\n    }�[0m\n    �[37m[17:07:31.138]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/cosmosContent\"\n    }�[0m\n    �[37m[17:07:31.148]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[17:07:42.926]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect\"\n    }�[0m\n    �[37m[17:07:43.042]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n      ]\n    }�[0m\n    �[37m[17:07:43.166]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j0325120-mgmt-2\",\n        \"ci01-j0325120-svc\"\n      ]\n    }�[0m\n    �[37m[17:07:43.202]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaces\": [\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n      ]\n    }�[0m\n    �[37m[17:07:43.202]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaceCount\": 2\n    }�[0m\n    �[37m[17:07:43.202]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n    }�[0m\n    �[37m[17:07:43.435]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n    }�[0m\n    �[37m[17:07:43.942]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:29.3103489Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\\\", ActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", SubActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \\u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188\"\n    }�[0m\n    �[37m[17:07:43.942]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:29.3103489Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\\\", ActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", SubActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \\u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[17:07:44.226]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200\"\n    }�[0m\n    �[37m[17:07:47.896]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:46.3200405Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\\\", ActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", SubActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \\u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[17:07:47.896]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 76/289 failed (KQL: let _startTime = datetime(2026-10-14T23:22:44.2253704Z);\\nlet _endTime = datetime(2026-10-14T23:27:44.2253703Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-14T23:22:44.2253704Z);\\nlet _endTime = datetime(2026-10-14T23:27:44.2253703Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[17:07:47.968]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/cosmosContent\"\n    }�[0m\n    �[37m[17:07:47.968]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[17:07:48.283]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 1106,\n      \"files\": 1\n    }�[0m\n    �[37m[17:07:48.283]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/cosmosContent\"\n    }�[0m\n    �[37m[17:07:48.293]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[17:08:00.494]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n      \"commits\": 613\n    }�[0m\n    �[37m[17:08:00.495]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect\"\n    }�[0m\n    �[37m[17:08:00.610]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n      ]\n    }�[0m\n    �[37m[17:08:00.715]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j0325120-mgmt-2\",\n        \"ci01-j0325120-svc\"\n      ]\n    }�[0m\n    �[37m[17:08:00.747]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaces\": [\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n      ]\n    }�[0m\n    �[37m[17:08:00.747]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaceCount\": 2\n    }�[0m\n    �[37m[17:08:00.747]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n    }�[0m\n    �[37m[17:08:01.068]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n    }�[0m\n    �[37m[17:08:01.547]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:46.3200405Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\\\", ActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", SubActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \\u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\\\"}, IsPermanent:false})\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200\"\n    }�[0m\n    �[37m[17:08:01.548]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:07:46.3200405Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\\\", ActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", SubActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \\u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\\\"}, IsPermanent:false})\"\n    }�[0m\n    \n      query-skip-hcp-and-events: command failed: exit status 1\n    output: �[37m[17:08:01.848]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533\"\n    }�[0m\n    �[37m[17:08:05.427]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:08:03.7881995Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\\\", ActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", SubActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \\u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[17:08:05.428]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 85/289 failed (KQL: let _startTime = datetime(2026-10-15T00:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T00:13:01.8473530Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-mgmt-2', 'ci01-j0325120-svc', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-15T00:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T00:13:01.8473530Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j0325120-mgmt-2', 'ci01-j0325120-svc', 'ci01-j0325120-mgmt-2')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[17:08:05.428]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/cosmosContent\"\n    }�[0m\n    �[37m[17:08:05.428]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[17:08:05.743]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 1106,\n      \"files\": 1\n    }�[0m\n    �[37m[17:08:05.743]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/cosmosContent\"\n    }�[0m\n    �[37m[17:08:05.752]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[17:08:15.101]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect\"\n    }�[0m\n    �[37m[17:08:15.221]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n      ]\n    }�[0m\n    �[37m[17:08:15.326]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j0325120-mgmt-2\",\n        \"ci01-j0325120-svc\"\n      ]\n    }�[0m\n    �[37m[17:08:15.357]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaces\": [\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n        \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n      ]\n    }�[0m\n    �[37m[17:08:15.357]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespaceCount\": 2\n    }�[0m\n    �[37m[17:08:15.357]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\"\n    }�[0m\n    �[37m[17:08:15.495]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j0325120-mgmt-2\",\n      \"namespace\": \"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\"\n    }�[0m\n    �[37m[17:08:15.976]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:08:03.7881995Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\\\", ActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", SubActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \\u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533\"\n    }�[0m\n    �[37m[17:08:15.977]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:08:03.7881995Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\\\", ActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", SubActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \\u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n    }�[0m\n    \n    {\n        s: \"must-gather CLI test failures:\\n  query-basic: command failed: exit status 1\\noutput: \\x1b[37m[17:06:46.277]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-824225575\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:06:50.310]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\\\\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:06:48.7179236Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\\\\\\\", ActivityId:\\\\\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\\\\\", SubActivityId:\\\\\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \\\\u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:06:50.310]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 78/289 failed (KQL: let _startTime = datetime(2026-10-14T23:31:46.2770422Z);\\\\nlet _endTime = datetime(2026-10-14T23:36:46.2770421Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:06:50.376]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-824225575/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-824225575/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:06:50.376]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-824225575/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:06:50.697]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 1106,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[17:06:50.697]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-824225575/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:06:50.706]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[17:07:03.236]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n  \\\"commits\\\": 613\\n}\\x1b[0m\\n\\x1b[37m[17:07:03.237]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:03.356]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:07:03.470]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j0325120-mgmt-2\\\",\\n    \\\"ci01-j0325120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:07:03.503]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaces\\\": [\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:07:03.503]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaceCount\\\": 2\\n}\\x1b[0m\\n\\x1b[37m[17:07:03.503]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:03.650]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-824225575/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:04.108]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\\\\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:06:48.7179236Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\\\\\\\", ActivityId:\\\\\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\\\\\", SubActivityId:\\\\\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \\\\u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-824225575\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:04.108]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:06:46.2770422Z);\\\\nlet _endTime = datetime(2026-10-15T17:06:46.2770425Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:06:48.7179236Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003\\\\\\\", ActivityId:\\\\\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\\\\\", SubActivityId:\\\\\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;029ee06e-6004-40df-8ee2-f9b8fab8e003 ARID=6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e \\\\u003e GW.Http.CallContext/6f10d14e-4ae9-4672-9ca8-f3ac61ee5a1e)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n  query-skip-hcp-logs: command failed: exit status 1\\noutput: \\x1b[37m[17:07:04.396]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:13.034]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\\\\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:07:11.3621455Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\\\\\\\", ActivityId:\\\\\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\\\\\", SubActivityId:\\\\\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \\\\u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:13.034]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 83/289 failed (KQL: let _startTime = datetime(2026-10-14T23:57:04.3961354Z);\\\\nlet _endTime = datetime(2026-10-15T00:02:04.3961353Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:13.071]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:13.072]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:13.393]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 1106,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[17:07:13.393]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:13.403]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[17:07:25.804]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n  \\\"commits\\\": 613\\n}\\x1b[0m\\n\\x1b[37m[17:07:25.805]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:25.912]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:07:26.021]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j0325120-mgmt-2\\\",\\n    \\\"ci01-j0325120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:07:26.141]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaces\\\": [\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:07:26.141]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaceCount\\\": 2\\n}\\x1b[0m\\n\\x1b[37m[17:07:26.141]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:26.291]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:26.780]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\\\\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:07:11.3621455Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\\\\\\\", ActivityId:\\\\\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\\\\\", SubActivityId:\\\\\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \\\\u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-3319694816\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:26.780]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:04.3961354Z);\\\\nlet _endTime = datetime(2026-10-15T17:07:04.3961363Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:07:11.3621455Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac\\\\\\\", ActivityId:\\\\\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\\\\\", SubActivityId:\\\\\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"2d73786e-0dab-4734-916c-277b95dfe6b3\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;1dd1fdc6-9d0c-4086-98e4-36bbd2d998ac ARID=2d73786e-0dab-4734-916c-277b95dfe6b3 \\\\u003e GW.Http.CallContext/2d73786e-0dab-4734-916c-277b95dfe6b3)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n  query-skip-kubernetes-events: command failed: exit status 1\\noutput: \\x1b[37m[17:07:27.073]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:30.812]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\\\\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:07:29.3103489Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\\\\\\\", ActivityId:\\\\\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\\\\\", SubActivityId:\\\\\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \\\\u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:30.812]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 76/289 failed (KQL: let _startTime = datetime(2026-10-14T23:22:27.0728956Z);\\\\nlet _endTime = datetime(2026-10-14T23:27:27.0728955Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-14T23:22:27.0728956Z);\\\\nlet _endTime = datetime(2026-10-14T23:27:27.0728955Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:30.813]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:30.813]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:31.138]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 1106,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[17:07:31.138]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:31.148]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[17:07:42.926]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:43.042]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:07:43.166]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j0325120-mgmt-2\\\",\\n    \\\"ci01-j0325120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:07:43.202]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaces\\\": [\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:07:43.202]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaceCount\\\": 2\\n}\\x1b[0m\\n\\x1b[37m[17:07:43.202]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:43.435]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:43.942]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\\\\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:07:29.3103489Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\\\\\\\", ActivityId:\\\\\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\\\\\", SubActivityId:\\\\\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \\\\u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-4187817188\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:43.942]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:27.0728956Z);\\\\nlet _endTime = datetime(2026-10-15T17:07:27.0728960Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:07:29.3103489Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0\\\\\\\", ActivityId:\\\\\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\\\\\", SubActivityId:\\\\\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;610774d0-c330-49a8-ab7d-7f09094955f0 ARID=e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9 \\\\u003e GW.Http.CallContext/e5ebca64-bb1e-4eaa-8f85-20ad9daa28c9)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\"\\n}\\x1b[0m\\n\\n  query-collect-systemd-logs: command failed: exit status 1\\noutput: \\x1b[37m[17:07:44.226]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:47.896]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\\\\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:07:46.3200405Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\\\\\\\", ActivityId:\\\\\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\\\\\", SubActivityId:\\\\\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \\\\u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:47.896]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 76/289 failed (KQL: let _startTime = datetime(2026-10-14T23:22:44.2253704Z);\\\\nlet _endTime = datetime(2026-10-14T23:27:44.2253703Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-14T23:22:44.2253704Z);\\\\nlet _endTime = datetime(2026-10-14T23:27:44.2253703Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j0325120-svc', 'ci01-j0325120-mgmt-2', 'ci01-j0325120-mgmt-2')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:47.968]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:47.968]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:48.283]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 1106,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[17:07:48.283]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:07:48.293]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[17:08:00.494]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n  \\\"commits\\\": 613\\n}\\x1b[0m\\n\\x1b[37m[17:08:00.495]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:00.610]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:08:00.715]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j0325120-mgmt-2\\\",\\n    \\\"ci01-j0325120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:08:00.747]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaces\\\": [\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:08:00.747]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaceCount\\\": 2\\n}\\x1b[0m\\n\\x1b[37m[17:08:00.747]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:01.068]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:01.547]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\\\\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:07:46.3200405Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\\\\\\\", ActivityId:\\\\\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\\\\\", SubActivityId:\\\\\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \\\\u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-2806811200\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:01.548]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:07:44.2253704Z);\\\\nlet _endTime = datetime(2026-10-15T17:07:44.2253708Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:07:46.3200405Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce\\\\\\\", ActivityId:\\\\\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\\\\\", SubActivityId:\\\\\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"66a3669f-0e2c-4860-86dc-f849756f632e\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;7fb190cd-76d0-42f3-8ad7-3f17a1e760ce ARID=66a3669f-0e2c-4860-86dc-f849756f632e \\\\u003e GW.Http.CallContext/66a3669f-0e2c-4860-86dc-f849756f632e)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n  query-skip-hcp-and-events: command failed: exit status 1\\noutput: \\x1b[37m[17:08:01.848]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:05.427]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\\\\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:08:03.7881995Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\\\\\\\", ActivityId:\\\\\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\\\\\", SubActivityId:\\\\\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \\\\u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:05.428]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 85/289 failed (KQL: let _startTime = datetime(2026-10-15T00:08:01.8473531Z);\\\\nlet _endTime = datetime(2026-10-15T00:13:01.8473530Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j0325120-mgmt-2', 'ci01-j0325120-svc', 'ci01-j0325120-mgmt-2')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-15T00:08:01.8473531Z);\\\\nlet _endTime = datetime(2026-10-15T00:13:01.8473530Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j0325120-mgmt-2', 'ci01-j0325120-svc', 'ci01-j0325120-mgmt-2')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:05.428]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:05.428]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:05.743]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 1106,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[17:08:05.743]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:05.752]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[17:08:15.101]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:15.221]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:08:15.326]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j0325120-mgmt-2\\\",\\n    \\\"ci01-j0325120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:08:15.357]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaces\\\": [\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n    \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[17:08:15.357]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespaceCount\\\": 2\\n}\\x1b[0m\\n\\x1b[37m[17:08:15.357]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:15.495]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j0325120-mgmt-2\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533/oc-adm-inspect/namespaces/ocm-arohcpci01-2tfb6ahivb6uh2sc3slj1pjrgkmgv90q-v4p9d8o3w4h5q5v\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:15.976]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\\\\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:08:03.7881995Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\\\\\\\", ActivityId:\\\\\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\\\\\", SubActivityId:\\\\\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \\\\u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-2861883533\\\"\\n}\\x1b[0m\\n\\x1b[37m[17:08:15.977]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-14T17:08:01.8473531Z);\\\\nlet _endTime = datetime(2026-10-15T17:08:01.8473535Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-qcdxq4xvtwqs'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-15T17:08:03.7881995Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;518d989b-6e77-41dd-815d-50d297655c28\\\\\\\", ActivityId:\\\\\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\\\\\", SubActivityId:\\\\\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"d19bda2c-8f67-41fb-bf81-27155da79740\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;518d989b-6e77-41dd-815d-50d297655c28 ARID=d19bda2c-8f67-41fb-bf81-27155da79740 \\\\u003e GW.Http.CallContext/d19bda2c-8f67-41fb-bf81-27155da79740)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\"\\n}\\x1b[0m\\n\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7564/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110824481948700672","occurred_at":"2026-10-15T20:05:48Z","signature_id":"d93271bc2d1e33c5756c062edb6e015986af09e31b03f0d4c66c7f18239ee00b","pr_number":7564,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110758849320325120","occurred_at":"2026-10-15T15:45:00Z","signature_id":"ffb44160ee1bf517c1425796ad5891019b5ffd9507f7fc661274b60a50af74b6","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d55a541fe54a611e130419c1f169b8c73c34a953ed35defd84c0a24bbe0a88e3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-cjhwhd\" with version 5.0.0-rc.5 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2dcd6debe660\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-cjhwhd: failed waiting for cluster=\"cluster-zstream-5-0-cjhwhd\" in resourcegroup=\"rg-zstream-upgrade-5-0-zbcrdfvtst44\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0fa13488-8ecd-48a9-95b8-c369b6566865\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0fa13488-8ecd-48a9-95b8-c369b6566865\",\n      \"name\": \"0fa13488-8ecd-48a9-95b8-c369b6566865\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T18:18:12.815750049Z\",\n      \"endTime\": \"2026-10-13T18:47:19.806209332Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0fa13488-8ecd-48a9-95b8-c369b6566865\n            correlation ID: d259ddfa-e906-4fbb-974b-3c977af82a32\n            client request ID: be895625-b480-4da8-a968-9e02baae7cef\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-cjhwhd\" with version 5.0.0-rc.5 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2dcd6debe660\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-cjhwhd: failed waiting for cluster=\"cluster-zstream-5-0-cjhwhd\" in resourcegroup=\"rg-zstream-upgrade-5-0-zbcrdfvtst44\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0fa13488-8ecd-48a9-95b8-c369b6566865\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0fa13488-8ecd-48a9-95b8-c369b6566865\",\n      \"name\": \"0fa13488-8ecd-48a9-95b8-c369b6566865\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T18:18:12.815750049Z\",\n      \"endTime\": \"2026-10-13T18:47:19.806209332Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0fa13488-8ecd-48a9-95b8-c369b6566865\n            correlation ID: d259ddfa-e906-4fbb-974b-3c977af82a32\n            client request ID: be895625-b480-4da8-a968-9e02baae7cef\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-dllvsm\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x28758e830200\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-dllvsm: failed waiting for cluster=\"cp-ystream-upgrade-4-21-dllvsm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-kqbmkzfdlvpv\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ec3c05-7e89-48a9-8588-f769d1d82955\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ec3c05-7e89-48a9-8588-f769d1d82955\",\n      \"name\": \"96ec3c05-7e89-48a9-8588-f769d1d82955\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T18:18:30.340236179Z\",\n      \"endTime\": \"2026-10-13T18:47:39.7699444Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ec3c05-7e89-48a9-8588-f769d1d82955\n            correlation ID: 8e81f893-e896-41de-bfb5-ecaf7ea09f96\n            client request ID: 8d55acf1-8d30-4678-b2f6-7eed037af357\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-dllvsm\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x28758e830200\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-dllvsm: failed waiting for cluster=\"cp-ystream-upgrade-4-21-dllvsm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-kqbmkzfdlvpv\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ec3c05-7e89-48a9-8588-f769d1d82955\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ec3c05-7e89-48a9-8588-f769d1d82955\",\n      \"name\": \"96ec3c05-7e89-48a9-8588-f769d1d82955\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T18:18:30.340236179Z\",\n      \"endTime\": \"2026-10-13T18:47:39.7699444Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ec3c05-7e89-48a9-8588-f769d1d82955\n            correlation ID: 8e81f893-e896-41de-bfb5-ecaf7ea09f96\n            client request ID: 8d55acf1-8d30-4678-b2f6-7eed037af357\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"be1b7e4b70ae40bb176b235bc31683c57b20ea13e1c6ce838e84dccb47cf9506","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"f76417099900daa68e51ebde43fef8462ff51d9cfbe2e05ce62c4518ce93529f","pr_number":7248,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-25T02:06:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-23-x7n4rm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1df325fdde20\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-candidate-4-23-x7n4rm-mgfdjq\" in resourcegroup=\"rg-candidate-4-23-25wqkbxjvqpm\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-23-25wqkbxjvqpm/providers/Microsoft.Resources/deployments/customer-infra-cluster-candidate-4-23-x7n4rm-mgfdjq/operationStatuses/08584096986588873020\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"InternalServerError\\\",\\r\\n        \\\"message\\\": \\\"An error occurred.\\\",\\r\\n        \\\"details\\\": []\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-23-25wqkbxjvqpm/providers/Microsoft.Resources/deployments/customer-infra-cluster-candidate-4-23-x7n4rm-mgfdjq/operationStatuses/08584096986588873020\n            correlation ID: 66a77b00-0c74-46a1-bab1-d73fd9784240\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-23-x7n4rm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1df325fdde20\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-candidate-4-23-x7n4rm-mgfdjq\" in resourcegroup=\"rg-candidate-4-23-25wqkbxjvqpm\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-23-25wqkbxjvqpm/providers/Microsoft.Resources/deployments/customer-infra-cluster-candidate-4-23-x7n4rm-mgfdjq/operationStatuses/08584096986588873020\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"InternalServerError\\\",\\r\\n        \\\"message\\\": \\\"An error occurred.\\\",\\r\\n        \\\"details\\\": []\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-23-25wqkbxjvqpm/providers/Microsoft.Resources/deployments/customer-infra-cluster-candidate-4-23-x7n4rm-mgfdjq/operationStatuses/08584096986588873020\n            correlation ID: 66a77b00-0c74-46a1-bab1-d73fd9784240\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-21-gpscnh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19351c9c25a0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-21-gpscnh-6d9mtn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-q2fxp7w8nsb4\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-21-q2fxp7w8nsb4/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-21-gpscnh-6d9mtn/operationStatuses/08584096986507070624\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"InternalServerError\\\",\\r\\n        \\\"message\\\": \\\"An error occurred.\\\",\\r\\n        \\\"details\\\": []\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-21-q2fxp7w8nsb4/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-21-gpscnh-6d9mtn/operationStatuses/08584096986507070624\n            correlation ID: 73cd0e3a-2347-4fb2-a937-dc27e30c4de2\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-21-gpscnh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19351c9c25a0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-21-gpscnh-6d9mtn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-q2fxp7w8nsb4\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-21-q2fxp7w8nsb4/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-21-gpscnh-6d9mtn/operationStatuses/08584096986507070624\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"InternalServerError\\\",\\r\\n        \\\"message\\\": \\\"An error occurred.\\\",\\r\\n        \\\"details\\\": []\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-21-q2fxp7w8nsb4/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-21-gpscnh-6d9mtn/operationStatuses/08584096986507070624\n            correlation ID: 73cd0e3a-2347-4fb2-a937-dc27e30c4de2\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"0bee45c13efb267807d223dc42fdb6b42774c050c8b11335bf9553e927e7a784","pr_number":7496,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"c9a49014768753cae5a2b15e7de520cc636773dabb5a420f9a78aef34c0146ac","pr_number":7496,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"ed30ba2b16f76586a1f98b0129dc60d2a7c651f09c53499a5a21b5f5dff0e754","failure_pattern":"IngressController loadBalancer config is nil","search_query":"IngressController loadBalancer config is nil","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:198]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x284568f46bc0\u003e: \n    IngressController loadBalancer config is nil\n    {\n        s: \"IngressController loadBalancer config is nil\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:198]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x284568f46bc0\u003e: \n    IngressController loadBalancer config is nil\n    {\n        s: \"IngressController loadBalancer config is nil\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:198]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x374a5e1794b0\u003e: \n    IngressController loadBalancer config is nil\n    {\n        s: \"IngressController loadBalancer config is nil\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:198]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x374a5e1794b0\u003e: \n    IngressController loadBalancer config is nil\n    {\n        s: \"IngressController loadBalancer config is nil\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110129974156464128","occurred_at":"2026-10-13T22:06:04Z","signature_id":"1dff6fd88e7b67ee0e3d3f634407f5c779eb894a4458935f87ace999c5c319de","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110129974156464128","occurred_at":"2026-10-13T22:06:04Z","signature_id":"91cd8349ac9ddd72fa8ca1ae3cea72c2db06bc446d6e997ab59539aff1b80c1e","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9310a4a1ba70b92dcd897ea7ccef6ff3d68afd4098649e3461e4758a9ed7df2f","failure_pattern":"VerifyIngressControllerScope(External) timed out after 5m0","search_query":"VerifyIngressControllerScope(External) timed out after 5m0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:199]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2c6abc8c6320\u003e: \n    VerifyIngressControllerScope(External) timed out after 5m0.002s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyIngressControllerScope(External) timed out after 5m0.002s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2c6abc8c62a0\u003e{\n            msg: \"failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x2c6abc8c6260\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:199]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2c6abc8c6320\u003e: \n    VerifyIngressControllerScope(External) timed out after 5m0.002s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyIngressControllerScope(External) timed out after 5m0.002s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2c6abc8c62a0\u003e{\n            msg: \"failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x2c6abc8c6260\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:199]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1ba452ab87c0\u003e: \n    VerifyIngressControllerScope(External) timed out after 5m0.006s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyIngressControllerScope(External) timed out after 5m0.006s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1ba452ab87a0\u003e{\n            msg: \"failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x1ba452ab8760\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:199]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1ba452ab87c0\u003e: \n    VerifyIngressControllerScope(External) timed out after 5m0.006s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyIngressControllerScope(External) timed out after 5m0.006s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1ba452ab87a0\u003e{\n            msg: \"failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x1ba452ab8760\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110235250528358400","occurred_at":"2026-10-14T05:04:24Z","signature_id":"7d84995ad741c9681c8c3c699502f42032152628033a5acce166137600455589","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110235250528358400","occurred_at":"2026-10-14T05:04:24Z","signature_id":"c488b2162ac9532ad63b94efbbfbaac3551ef53bea0886a3d47dcc0d5f92dd8e","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f3a7b19a161f6d450e990aabc7d01a1298ee43d7b8bea6fe710061141d7aca91","failure_pattern":"All ClusterOperators should report Available=True, but these are not available: [console (Available=False)]","search_query":"All ClusterOperators should report Available=True, but these are not available: [console (Available=False)]","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:378]: Timed out after 639.547s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:377 with:\nAll ClusterOperators should report Available=True, but these are not available: [console (Available=False)]\nExpected\n    \u003c[]string | len:1, cap:1\u003e: [\n        \"console (Available=False)\",\n    ]\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:378]: Timed out after 639.547s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:377 with:\nAll ClusterOperators should report Available=True, but these are not available: [console (Available=False)]\nExpected\n    \u003c[]string | len:1, cap:1\u003e: [\n        \"console (Available=False)\",\n    ]\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110412598238253056","occurred_at":"2026-10-14T16:49:06Z","signature_id":"e27d6f0c138773167813ece164429b7c33de4079b9a5f054e3aa5a8fdc5f3e0f","pr_number":7529,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d9cf71c84b551be1ecbe865f510398e2d73ebb5ce47aadae2c80a4aba3f25f6a","failure_pattern":"Cluster GET returned error: Get \"\u003curl\u003e","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:116]: Cluster GET returned error: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-credential-lifecycle-test-zzgk4tx5xdlx?api-version","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:116]: Cluster GET returned error: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-credential-lifecycle-test-zzgk4tx5xdlx?api-version=2021-04-01\": stream error: stream ID 53; CANCEL; received from peer\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:116]: Cluster GET returned error: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-credential-lifecycle-test-zzgk4tx5xdlx?api-version=2021-04-01\": stream error: stream ID 53; CANCEL; received from peer"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110728083584585728","occurred_at":"2026-10-15T13:42:45Z","signature_id":"d42ee325c6e254f8a72276d7aaee39f3ec619484c2e7b4de617b1084e9af7fd6","pr_number":7387,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"96d801deab41ec98748c7082577ee746091d92728a5e786bd4eef6a4d1b2ca0f","failure_pattern":"DataCurrent condition must be True","search_query":"DataCurrent condition must be True","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Fleet should have HCP resource requirements data","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-22T12:46:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Fleet should have HCP resource requirements data","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:115]: Timed out after 1800.000s.\nHCP resource requirements were not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:105 with:\nDataCurrent condition must be True\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:115]: Timed out after 1800.000s.\nHCP resource requirements were not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:105 with:\nDataCurrent condition must be True\nExpected\n    \u003cbool\u003e: false\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"f4be36cc0230ad342fd316be591c2129d772f7846989d11fc93f73065bc1f045","pr_number":6982,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fa65c8375667d2f86c4c029a1dd32730b8452c4c21edd3768c8aaedc846a9603","failure_pattern":"ERROR CODE: Canceled; detail message This operation was superseded by another; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Canceled","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-19T17:39:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-mpppvh with version 4.21.37\nUnexpected error:\n    \u003c*fmt.wrapError | 0x279027f833e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-mpppvh: failed waiting for cluster=\"np-version-upgrade-cluster-mpppvh\" in resourcegroup=\"rg-np-version-upgrade-mpppvh-6ckrm5njjkhq\" to finish creating: GET https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7ef9bc0-8329-437a-9805-6b4b65fd8b47\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: Canceled\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7ef9bc0-8329-437a-9805-6b4b65fd8b47\",\n      \"name\": \"a7ef9bc0-8329-437a-9805-6b4b65fd8b47\",\n      \"status\": \"Canceled\",\n      \"startTime\": \"2026-10-15T04:31:31.170202136Z\",\n      \"endTime\": \"2026-10-15T04:31:34.719444109Z\",\n      \"error\": {\n        \"code\": \"Canceled\",\n        \"message\": \"This operation was superseded by another\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7ef9bc0-8329-437a-9805-6b4b65fd8b47\n            correlation ID: bbacf445-8423-4177-9af5-a54728857aa3\n            client request ID: 33a84f8d-a321-43e3-a649-cc65e487871e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-mpppvh with version 4.21.37\nUnexpected error:\n    \u003c*fmt.wrapError | 0x279027f833e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-mpppvh: failed waiting for cluster=\"np-version-upgrade-cluster-mpppvh\" in resourcegroup=\"rg-np-version-upgrade-mpppvh-6ckrm5njjkhq\" to finish creating: GET https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7ef9bc0-8329-437a-9805-6b4b65fd8b47\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: Canceled\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7ef9bc0-8329-437a-9805-6b4b65fd8b47\",\n      \"name\": \"a7ef9bc0-8329-437a-9805-6b4b65fd8b47\",\n      \"status\": \"Canceled\",\n      \"startTime\": \"2026-10-15T04:31:31.170202136Z\",\n      \"endTime\": \"2026-10-15T04:31:34.719444109Z\",\n      \"error\": {\n        \"code\": \"Canceled\",\n        \"message\": \"This operation was superseded by another\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7ef9bc0-8329-437a-9805-6b4b65fd8b47\n            correlation ID: bbacf445-8423-4177-9af5-a54728857aa3\n            client request ID: 33a84f8d-a321-43e3-a649-cc65e487871e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7401/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110572658033692672","occurred_at":"2026-10-15T03:25:08Z","signature_id":"3463f8e6114f7652dff0d4ce4c5cc8eb355bff4de07583420b62c0e190564e47","pr_number":7401,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1019af4c9c47d1aa706702f78998f006a0e49b021ad920d44d356df2637fa18e","failure_pattern":"ERROR CODE: Conflict; detail message Cannot update resource while resource is \"accepted\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-19T17:39:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-mpppvh with version 4.21.37\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1bc8b1824920\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-mpppvh: failed starting cluster creation \"np-version-upgrade-cluster-mpppvh\" in resourcegroup=\"rg-np-version-upgrade-mpppvh-6ckrm5njjkhq\": PUT https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mpppvh-6ckrm5njjkhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mpppvh\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot update resource while resource is \\\"accepted\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mpppvh-6ckrm5njjkhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mpppvh\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mpppvh-6ckrm5njjkhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mpppvh\n            correlation ID: 5672e540-0082-400e-8d95-fd1926fa87e0\n            client request ID: 32f4cab2-abf5-42a2-aa1e-6a050c192397\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-mpppvh with version 4.21.37\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1bc8b1824920\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-mpppvh: failed starting cluster creation \"np-version-upgrade-cluster-mpppvh\" in resourcegroup=\"rg-np-version-upgrade-mpppvh-6ckrm5njjkhq\": PUT https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mpppvh-6ckrm5njjkhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mpppvh\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot update resource while resource is \\\"accepted\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mpppvh-6ckrm5njjkhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mpppvh\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://rp.j3692672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mpppvh-6ckrm5njjkhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mpppvh\n            correlation ID: 5672e540-0082-400e-8d95-fd1926fa87e0\n            client request ID: 32f4cab2-abf5-42a2-aa1e-6a050c192397\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7401/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110572658033692672","occurred_at":"2026-10-15T03:25:08Z","signature_id":"3747b30e570e4b1c00673a1f97fdb92c7523478b496e1ded3b8cf9cde8d49d23","pr_number":7401,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"22fbdb35a870cfc5f2c9e8846465e0b7f85e96d6e430537b6f1feed38d45976b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code KeyVaultAccessForbidden; detail message Unable to access key vault resource '\u003curl\u003e' to enable encryption at rest. Please grant get, wrap and unwrap key permissions to disk encryption set 'des... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x14dd1e091060\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-kdz84x79jt45\" to finish creating: GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\",\n      \"name\": \"75f3427d-1a2b-4425-b9b0-7480d4e0b89b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T12:59:06.888844222Z\",\n      \"endTime\": \"2026-10-12T13:28:08.456404461Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine des-encrypt-des-np-f8895-79vxf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine des-encrypt-des-np-f8895-gfvrv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine des-encrypt-des-np-f8895-79vxf: Failed: virtualmachine failed to create or update. err: failed to create or update resource des-encrypt-kdz84x79jt45--managed/des-encrypt-des-np-f8895-79vxf (service: virtualmachine): PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-kdz84x79jt45--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-f8895-79vxf\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: KeyVaultAccessForbidden\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"KeyVaultAccessForbidden\\\",\\n    \\\"message\\\": \\\"Unable to access key vault resource 'https://cust-kv-knxkedtao2ry2.vault.azure.net/keys/des-encryption-key/3126b361c5834ce5ad1e97723eb63855' to enable encryption at rest. Please grant get, wrap and unwrap key permissions to disk encryption set 'des... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\n            correlation ID: bd8a92be-4962-42e7-8e34-639e73d20045\n            client request ID: 6eda585f-7541-486b-8dad-d5cd134191ff\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x14dd1e091060\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-kdz84x79jt45\" to finish creating: GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\",\n      \"name\": \"75f3427d-1a2b-4425-b9b0-7480d4e0b89b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T12:59:06.888844222Z\",\n      \"endTime\": \"2026-10-12T13:28:08.456404461Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine des-encrypt-des-np-f8895-79vxf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine des-encrypt-des-np-f8895-gfvrv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine des-encrypt-des-np-f8895-79vxf: Failed: virtualmachine failed to create or update. err: failed to create or update resource des-encrypt-kdz84x79jt45--managed/des-encrypt-des-np-f8895-79vxf (service: virtualmachine): PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-kdz84x79jt45--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-f8895-79vxf\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: KeyVaultAccessForbidden\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"KeyVaultAccessForbidden\\\",\\n    \\\"message\\\": \\\"Unable to access key vault resource 'https://cust-kv-knxkedtao2ry2.vault.azure.net/keys/des-encryption-key/3126b361c5834ce5ad1e97723eb63855' to enable encryption at rest. Please grant get, wrap and unwrap key permissions to disk encryption set 'des... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\n            correlation ID: bd8a92be-4962-42e7-8e34-639e73d20045\n            client request ID: 6eda585f-7541-486b-8dad-d5cd134191ff\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"a902cb53b68f347d3389b746e7a2281dfe680f58c4b7f49f68ca93a281c00759","pr_number":7470,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"eb83b58a2fadd83fd240e7d029c84105051148d1e8c5634b73b310d5d0e53edc","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please ... message truncated Machine \u003cnode\u003e: NotReady: AzureMachine status.initialization.provisioned is false; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a4934179d80\u003e: \n    failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-gtft8mgc5kwc\" to finish creating: GET https://rp.j5701632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37156461-5409-48c2-85c8-26e90a95c4af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37156461-5409-48c2-85c8-26e90a95c4af\",\n      \"name\": \"37156461-5409-48c2-85c8-26e90a95c4af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T20:27:24.635428507Z\",\n      \"endTime\": \"2026-10-15T20:46:31.281302981Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ephemeral-disk-ephemeral-np-ghcfv-62gf2: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ephemeral-disk-ephemeral-np-ghcfv-n2f47: Failed: availabilitysets failed to create or update. err: failed to get existing resource ephemeral-osdisk-gtft8mgc5kwc--managed/2tfeeaoo9r717nalfklm07mkpjq3ca2e_ephemeral-disk-ephemeral-np-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-gtft8mgc5kwc--managed/providers/Microsoft.Compute/availabilitySets/2tfeeaoo9r717nalfklm07mkpjq3ca2e_ephemeral-disk-ephemeral-np-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '8d1ff6d0-aadd-4331-9171-9443ac5f4337' exceeded. Please ... message truncated\\nMachine ephemeral-disk-ephemeral-np-ghcfv-62gf2: NotReady: AzureMachine status.initialization.provisioned is false\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5701632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37156461-5409-48c2-85c8-26e90a95c4af\n            correlation ID: 44913fa3-dfaa-4300-8f6a-77ee9b21e91f\n            client request ID: b52ff7de-d76a-4d6c-8d81-14684e59505a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a4934179d80\u003e: \n    failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-gtft8mgc5kwc\" to finish creating: GET https://rp.j5701632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37156461-5409-48c2-85c8-26e90a95c4af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37156461-5409-48c2-85c8-26e90a95c4af\",\n      \"name\": \"37156461-5409-48c2-85c8-26e90a95c4af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T20:27:24.635428507Z\",\n      \"endTime\": \"2026-10-15T20:46:31.281302981Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ephemeral-disk-ephemeral-np-ghcfv-62gf2: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ephemeral-disk-ephemeral-np-ghcfv-n2f47: Failed: availabilitysets failed to create or update. err: failed to get existing resource ephemeral-osdisk-gtft8mgc5kwc--managed/2tfeeaoo9r717nalfklm07mkpjq3ca2e_ephemeral-disk-ephemeral-np-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-gtft8mgc5kwc--managed/providers/Microsoft.Compute/availabilitySets/2tfeeaoo9r717nalfklm07mkpjq3ca2e_ephemeral-disk-ephemeral-np-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '8d1ff6d0-aadd-4331-9171-9443ac5f4337' exceeded. Please ... message truncated\\nMachine ephemeral-disk-ephemeral-np-ghcfv-62gf2: NotReady: AzureMachine status.initialization.provisioned is false\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5701632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37156461-5409-48c2-85c8-26e90a95c4af\n            correlation ID: 44913fa3-dfaa-4300-8f6a-77ee9b21e91f\n            client request ID: b52ff7de-d76a-4d6c-8d81-14684e59505a\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110810477125701632","occurred_at":"2026-10-15T19:10:09Z","signature_id":"0658e0a4e1e57b932b4cb3283f6e5e5d242b0b18e06ad3346798a94bfc70208a","pr_number":7560,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"6494a57235b1e5ea0036c9f08ed717c4d99f3e06862fbe4cb98568bb3fb00c61","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after a... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2dbe92ada858\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-t6jslt7lwjkt\" to finish creating: GET https://rp.j2382080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/88e58151-a475-4725-8e82-2d51b0f7fb69\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/88e58151-a475-4725-8e82-2d51b0f7fb69\",\n      \"name\": \"88e58151-a475-4725-8e82-2d51b0f7fb69\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T13:50:29.440992073Z\",\n      \"endTime\": \"2026-10-16T14:09:32.07987309Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-5p4h6-brpt4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-t6jslt7lwjkt--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-5p4h6-brpt4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-5p4h6-brpt4: Failed: availabilitysets failed to create or update. err: failed to get existing resource cni-cilium-t6jslt7lwjkt--managed/2tftmlmmdno0igptnhfrk6md9uivpr81_cilium-cl-cilium-np-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-t6jslt7lwjkt--managed/providers/Microsoft.Compute/availabilitySets/2tftmlmmdno0igptnhfrk6md9uivpr81_cilium-cl-cilium-np-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '10' seconds after a... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2382080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/88e58151-a475-4725-8e82-2d51b0f7fb69\n            correlation ID: 054f0541-4905-4acf-9baf-353f2e747fdd\n            client request ID: 320b7448-1575-4fdb-9611-9bf748713191\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2dbe92ada858\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-t6jslt7lwjkt\" to finish creating: GET https://rp.j2382080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/88e58151-a475-4725-8e82-2d51b0f7fb69\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/88e58151-a475-4725-8e82-2d51b0f7fb69\",\n      \"name\": \"88e58151-a475-4725-8e82-2d51b0f7fb69\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T13:50:29.440992073Z\",\n      \"endTime\": \"2026-10-16T14:09:32.07987309Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-5p4h6-brpt4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-t6jslt7lwjkt--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-5p4h6-brpt4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-5p4h6-brpt4: Failed: availabilitysets failed to create or update. err: failed to get existing resource cni-cilium-t6jslt7lwjkt--managed/2tftmlmmdno0igptnhfrk6md9uivpr81_cilium-cl-cilium-np-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-t6jslt7lwjkt--managed/providers/Microsoft.Compute/availabilitySets/2tftmlmmdno0igptnhfrk6md9uivpr81_cilium-cl-cilium-np-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '10' seconds after a... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2382080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/88e58151-a475-4725-8e82-2d51b0f7fb69\n            correlation ID: 054f0541-4905-4acf-9baf-353f2e747fdd\n            client request ID: 320b7448-1575-4fdb-9611-9bf748713191\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111079254782382080","occurred_at":"2026-10-16T12:58:10Z","signature_id":"f1597e364c1f86f7d3afaa02ce110c4da89f5136194711d5b0ee2274d8ec6008","pr_number":7556,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"3a08509cc4313fdb7de088f97578445c52be0514ee3ea35913c562d65fc01af4","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. ... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2fa3cd2a3c20\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-wl8xt5tlkcq7\" to finish creating: GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/63825d69-d610-4421-b3ee-50ac654908c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/63825d69-d610-4421-b3ee-50ac654908c5\",\n      \"name\": \"63825d69-d610-4421-b3ee-50ac654908c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T21:00:14.715888281Z\",\n      \"endTime\": \"2026-10-16T21:19:17.454369198Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-rnlcf-ttbgr: Failed: interfaces failed to create or update. err: failed to get existing resource external-auth-cluster-wl8xt5tlkcq7--managed/ea-cluster-ea-np-1-rnlcf-ttbgr-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-wl8xt5tlkcq7--managed/providers/Microsoft.Network/networkInterfaces/ea-cluster-ea-np-1-rnlcf-ttbgr-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Please try again after '10' seconds after additional tokens are available. ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/63825d69-d610-4421-b3ee-50ac654908c5\n            correlation ID: 2667b7c1-2456-4fb3-a1b3-9f090199ed55\n            client request ID: 1483f922-ab7c-4902-933a-1a3aef1cf957\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2fa3cd2a3c20\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-wl8xt5tlkcq7\" to finish creating: GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/63825d69-d610-4421-b3ee-50ac654908c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/63825d69-d610-4421-b3ee-50ac654908c5\",\n      \"name\": \"63825d69-d610-4421-b3ee-50ac654908c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T21:00:14.715888281Z\",\n      \"endTime\": \"2026-10-16T21:19:17.454369198Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-rnlcf-ttbgr: Failed: interfaces failed to create or update. err: failed to get existing resource external-auth-cluster-wl8xt5tlkcq7--managed/ea-cluster-ea-np-1-rnlcf-ttbgr-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-wl8xt5tlkcq7--managed/providers/Microsoft.Network/networkInterfaces/ea-cluster-ea-np-1-rnlcf-ttbgr-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Please try again after '10' seconds after additional tokens are available. ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/63825d69-d610-4421-b3ee-50ac654908c5\n            correlation ID: 2667b7c1-2456-4fb3-a1b3-9f090199ed55\n            client request ID: 1483f922-ab7c-4902-933a-1a3aef1cf957\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7531/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111186341810147328","occurred_at":"2026-10-16T20:03:41Z","signature_id":"0270ee34b1d1960ce57816d9c23f2bf4723eabf324cf85fb883ca827cb1fc8d2","pr_number":7531,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1a35aeac2b02d0bb0965498ad4bf13cb371d32eb6b72e0ddbe98ebe03cb92ab3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. Refer ... message truncated Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33c730fe0780\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-2q8nt57gw7c4\" to finish creating: GET https://rp.j3035776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f78025-9771-4e07-9a76-cc316ad5bcaa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f78025-9771-4e07-9a76-cc316ad5bcaa\",\n      \"name\": \"f4f78025-9771-4e07-9a76-cc316ad5bcaa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T23:21:44.60401103Z\",\n      \"endTime\": \"2026-10-16T23:40:48.245826795Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p9v1y5n5j8r9i0x-np-1-89jk5-zffrv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2q8nt57gw7c4--managed/providers/Microsoft.Compute/virtualMachines/p9v1y5n5j8r9i0x-np-1-89jk5-zffrv to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p9v1y5n5j8r9i0x-np-1-89jk5-vpf5w: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-2q8nt57gw7c4--managed/p9v1y5n5j8r9i0x-np-1-89jk5-vpf5w-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2q8nt57gw7c4--managed/providers/Microsoft.Network/networkInterfaces/p9v1y5n5j8r9i0x-np-1-89jk5-vpf5w-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3d8c36e1-ae7a-42ef-b7d1-ea4667708d30' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\nMachine p9v1y5n5j8r9i0x-np-1-89jk5-zffrv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2q8nt57gw7c4--managed/providers/Microsoft.Compute/virtualMachines/p9v1y5n5j8r9i0x-np-1-89jk5-zffrv to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3035776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f78025-9771-4e07-9a76-cc316ad5bcaa\n            correlation ID: 57a29177-39d2-4d3f-ac56-44b9805e57f9\n            client request ID: 8065bb9a-6902-4747-a2b4-2914d4dc8e70\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33c730fe0780\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-2q8nt57gw7c4\" to finish creating: GET https://rp.j3035776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f78025-9771-4e07-9a76-cc316ad5bcaa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f78025-9771-4e07-9a76-cc316ad5bcaa\",\n      \"name\": \"f4f78025-9771-4e07-9a76-cc316ad5bcaa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T23:21:44.60401103Z\",\n      \"endTime\": \"2026-10-16T23:40:48.245826795Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p9v1y5n5j8r9i0x-np-1-89jk5-zffrv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2q8nt57gw7c4--managed/providers/Microsoft.Compute/virtualMachines/p9v1y5n5j8r9i0x-np-1-89jk5-zffrv to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p9v1y5n5j8r9i0x-np-1-89jk5-vpf5w: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-2q8nt57gw7c4--managed/p9v1y5n5j8r9i0x-np-1-89jk5-vpf5w-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2q8nt57gw7c4--managed/providers/Microsoft.Network/networkInterfaces/p9v1y5n5j8r9i0x-np-1-89jk5-vpf5w-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3d8c36e1-ae7a-42ef-b7d1-ea4667708d30' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\nMachine p9v1y5n5j8r9i0x-np-1-89jk5-zffrv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2q8nt57gw7c4--managed/providers/Microsoft.Compute/virtualMachines/p9v1y5n5j8r9i0x-np-1-89jk5-zffrv to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3035776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f78025-9771-4e07-9a76-cc316ad5bcaa\n            correlation ID: 57a29177-39d2-4d3f-ac56-44b9805e57f9\n            client request ID: 8065bb9a-6902-4747-a2b4-2914d4dc8e70\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7531/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111218975433035776","occurred_at":"2026-10-16T22:13:22Z","signature_id":"2bf01ccbacd9feffa73eddf299b329d411264d68e14c66906bd09f4e0f7f7b5d","pr_number":7531,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b7a7316d7bc9a13546759ba543c0393779c1adcbaf7eafe296163143bdb55b2c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. Refer ... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33cb56e17020\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4xcd2z54mnjl\" to finish creating: GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed1bfb49-b931-4647-a8ac-58b583c3b50b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed1bfb49-b931-4647-a8ac-58b583c3b50b\",\n      \"name\": \"ed1bfb49-b931-4647-a8ac-58b583c3b50b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T10:16:52.135966436Z\",\n      \"endTime\": \"2026-10-16T10:35:53.058795763Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s2n7n1o2s1t5d5z-np-1-m5qjf-pw27v: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine s2n7n1o2s1t5d5z-np-1-m5qjf-49zn4: Failed: virtualmachine failed to create or update. err: failed to get existing resource private-keyvault-4xcd2z54mnjl--managed/s2n7n1o2s1t5d5z-np-1-m5qjf-49zn4 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xcd2z54mnjl--managed/providers/Microsoft.Compute/virtualMachines/s2n7n1o2s1t5d5z-np-1-m5qjf-49zn4\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3871b527-fb1e-4123-b38b-3cb2445a9fc8' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed1bfb49-b931-4647-a8ac-58b583c3b50b\n            correlation ID: ef5c7325-64f6-4ca1-87ca-c2c3e96bf69a\n            client request ID: 392329dd-b2a3-4225-a4e8-154f2f223edb\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33cb56e17020\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4xcd2z54mnjl\" to finish creating: GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed1bfb49-b931-4647-a8ac-58b583c3b50b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed1bfb49-b931-4647-a8ac-58b583c3b50b\",\n      \"name\": \"ed1bfb49-b931-4647-a8ac-58b583c3b50b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T10:16:52.135966436Z\",\n      \"endTime\": \"2026-10-16T10:35:53.058795763Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s2n7n1o2s1t5d5z-np-1-m5qjf-pw27v: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine s2n7n1o2s1t5d5z-np-1-m5qjf-49zn4: Failed: virtualmachine failed to create or update. err: failed to get existing resource private-keyvault-4xcd2z54mnjl--managed/s2n7n1o2s1t5d5z-np-1-m5qjf-49zn4 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xcd2z54mnjl--managed/providers/Microsoft.Compute/virtualMachines/s2n7n1o2s1t5d5z-np-1-m5qjf-49zn4\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3871b527-fb1e-4123-b38b-3cb2445a9fc8' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed1bfb49-b931-4647-a8ac-58b583c3b50b\n            correlation ID: ef5c7325-64f6-4ca1-87ca-c2c3e96bf69a\n            client request ID: 392329dd-b2a3-4225-a4e8-154f2f223edb\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7574/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111024523175268352","occurred_at":"2026-10-16T09:20:41Z","signature_id":"884768de414f2c6367d7336de5c7997f81daf783ffd0d385d67afc2eda8eab1a","pr_number":7574,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"445bea11baaa5e6a0f158d83698b151143249148184d0e3c92735ff6d2548efa","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x172a79b02020\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-tjmbtfqzrk9p\" to finish creating: GET https://rp.j5936768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/166953ee-9ecb-476c-9dcf-5d9a18601d55\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/166953ee-9ecb-476c-9dcf-5d9a18601d55\",\n      \"name\": \"166953ee-9ecb-476c-9dcf-5d9a18601d55\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T23:11:14.489775395Z\",\n      \"endTime\": \"2026-10-15T23:30:19.374154808Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m5d0c0m5m4r5p9d-np-1-2f6qw-lqwqf: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-tjmbtfqzrk9p--managed/m5d0c0m5m4r5p9d-np-1-2f6qw-lqwqf-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-tjmbtfqzrk9p--managed/providers/Microsoft.Network/networkInterfaces/m5d0c0m5m4r5p9d-np-1-2f6qw-lqwqf-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '3' seconds after additional tokens are available.... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5936768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/166953ee-9ecb-476c-9dcf-5d9a18601d55\n            correlation ID: 0f13c63d-8045-459d-a9cd-c324c682e717\n            client request ID: a95fa974-534b-4390-90e8-ad21cd86c818\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x172a79b02020\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-tjmbtfqzrk9p\" to finish creating: GET https://rp.j5936768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/166953ee-9ecb-476c-9dcf-5d9a18601d55\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/166953ee-9ecb-476c-9dcf-5d9a18601d55\",\n      \"name\": \"166953ee-9ecb-476c-9dcf-5d9a18601d55\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T23:11:14.489775395Z\",\n      \"endTime\": \"2026-10-15T23:30:19.374154808Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m5d0c0m5m4r5p9d-np-1-2f6qw-lqwqf: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-tjmbtfqzrk9p--managed/m5d0c0m5m4r5p9d-np-1-2f6qw-lqwqf-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-tjmbtfqzrk9p--managed/providers/Microsoft.Network/networkInterfaces/m5d0c0m5m4r5p9d-np-1-2f6qw-lqwqf-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '3' seconds after additional tokens are available.... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5936768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/166953ee-9ecb-476c-9dcf-5d9a18601d55\n            correlation ID: 0f13c63d-8045-459d-a9cd-c324c682e717\n            client request ID: a95fa974-534b-4390-90e8-ad21cd86c818\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110856070715936768","occurred_at":"2026-10-15T22:11:19Z","signature_id":"7070accd24a6e67405d63ffa90a1616d2e2ec80da3d47eb48ef1723b01ff9253","pr_number":7560,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"64c6e11cbb74bffe980698f0ac18939b0cc44bb682a44aa8b5b4f43aee00cd17","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-mvkgs4\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2777a0f273c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-mvkgs4: failed waiting for cluster=\"cp-ystream-upgrade-4-22-mvkgs4\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-blssk6qzfvws\" to finish creating: GET https://rp.j4380032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\",\n      \"name\": \"a49f2790-97bb-4d56-b4a6-e43cb33e230e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T15:52:53.687204284Z\",\n      \"endTime\": \"2026-10-12T16:21:54.576185121Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, cluster-policy-controller, kube-controller-manager, openshift-apiserver, cluster-image-registry-operator, redhat-operators-catalog, community-operators-catalog, openshift-controller-manager, konnectivity-agent, olm-operator, packageserver, certified-operators-catalog, cluster-version-operator, csi-snapshot-controller-operator, catalog-operator, kube-scheduler, cluster-node-tuning-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4380032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\n            correlation ID: f4045377-6462-4996-b798-a9fd2a096fae\n            client request ID: 01fbf192-7565-417e-a2b4-3d30deb4b951\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-mvkgs4\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2777a0f273c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-mvkgs4: failed waiting for cluster=\"cp-ystream-upgrade-4-22-mvkgs4\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-blssk6qzfvws\" to finish creating: GET https://rp.j4380032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\",\n      \"name\": \"a49f2790-97bb-4d56-b4a6-e43cb33e230e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T15:52:53.687204284Z\",\n      \"endTime\": \"2026-10-12T16:21:54.576185121Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, cluster-policy-controller, kube-controller-manager, openshift-apiserver, cluster-image-registry-operator, redhat-operators-catalog, community-operators-catalog, openshift-controller-manager, konnectivity-agent, olm-operator, packageserver, certified-operators-catalog, cluster-version-operator, csi-snapshot-controller-operator, catalog-operator, kube-scheduler, cluster-node-tuning-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4380032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\n            correlation ID: f4045377-6462-4996-b798-a9fd2a096fae\n            client request ID: 01fbf192-7565-417e-a2b4-3d30deb4b951\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109662114334380032","occurred_at":"2026-10-12T15:06:56Z","signature_id":"93e4587d665844128088f4ce3a0a7ac0670bc2eccb4898d241b67e1ee37e48d9","pr_number":7383,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2c0a88f2a0ef84e4c12e30b05c3779aee6f0cf66551b88fac3c1693eccdbecc5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-29T01:37:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d5895186200\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-knlckq8m42ph\" to finish creating: GET https://rp.j7016064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b31da953-f506-41e7-9fc1-8b942d4e8068\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b31da953-f506-41e7-9fc1-8b942d4e8068\",\n      \"name\": \"b31da953-f506-41e7-9fc1-8b942d4e8068\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T21:44:40.287298623Z\",\n      \"endTime\": \"2026-10-14T22:13:42.52978598Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-network-operator, catalog-operator, packageserver, openshift-route-controller-manager, cluster-policy-controller, cluster-storage-operator, control-plane-pki-operator, openshift-controller-manager, dns-operator, kube-controller-manager, cluster-node-tuning-operator, olm-operator, redhat-operators-catalog, certified-operators-catalog, csi-snapshot-controller-operator, machine-approver, azure-cloud-controller-manager, cluster-autoscaler, openshift-apiserver, hosted-cluster-config-operator, redhat-marketplace-catalog, community-operators-catalog, cluster-version-operator, ingress-operator, cluster-image-registry-operator, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7016064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b31da953-f506-41e7-9fc1-8b942d4e8068\n            correlation ID: 4b021430-3a3e-4288-80c7-fd56abc38462\n            client request ID: bf735dc6-c8ef-42d7-978e-a9ed4b21e87f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d5895186200\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-knlckq8m42ph\" to finish creating: GET https://rp.j7016064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b31da953-f506-41e7-9fc1-8b942d4e8068\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b31da953-f506-41e7-9fc1-8b942d4e8068\",\n      \"name\": \"b31da953-f506-41e7-9fc1-8b942d4e8068\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T21:44:40.287298623Z\",\n      \"endTime\": \"2026-10-14T22:13:42.52978598Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-network-operator, catalog-operator, packageserver, openshift-route-controller-manager, cluster-policy-controller, cluster-storage-operator, control-plane-pki-operator, openshift-controller-manager, dns-operator, kube-controller-manager, cluster-node-tuning-operator, olm-operator, redhat-operators-catalog, certified-operators-catalog, csi-snapshot-controller-operator, machine-approver, azure-cloud-controller-manager, cluster-autoscaler, openshift-apiserver, hosted-cluster-config-operator, redhat-marketplace-catalog, community-operators-catalog, cluster-version-operator, ingress-operator, cluster-image-registry-operator, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7016064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b31da953-f506-41e7-9fc1-8b942d4e8068\n            correlation ID: 4b021430-3a3e-4288-80c7-fd56abc38462\n            client request ID: bf735dc6-c8ef-42d7-978e-a9ed4b21e87f\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7527/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110474746717016064","occurred_at":"2026-10-14T20:56:04Z","signature_id":"48f5d6fe28a1e01e349727dd8c1e637671c49c3f0a083af8d28b1fa2eadaaa1d","pr_number":7527,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"727b90d738458fae42fd743aac06537d1157ba87ee4430dc05269762a9e0b6d7","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0x384da2442800\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-65x7ldfvpfrb\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d7260b8-3447-49f3-b1d1-85b028cd4875\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d7260b8-3447-49f3-b1d1-85b028cd4875\",\n      \"name\": \"8d7260b8-3447-49f3-b1d1-85b028cd4875\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T18:19:32.972571872Z\",\n      \"endTime\": \"2026-10-13T18:48:39.815141519Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d7260b8-3447-49f3-b1d1-85b028cd4875\n            correlation ID: 7e9c28a1-2d97-4241-837c-65dbd7be5ebd\n            client request ID: 76ea1b85-c87c-48c0-b8b1-76222a78e914\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0x384da2442800\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-65x7ldfvpfrb\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d7260b8-3447-49f3-b1d1-85b028cd4875\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d7260b8-3447-49f3-b1d1-85b028cd4875\",\n      \"name\": \"8d7260b8-3447-49f3-b1d1-85b028cd4875\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T18:19:32.972571872Z\",\n      \"endTime\": \"2026-10-13T18:48:39.815141519Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d7260b8-3447-49f3-b1d1-85b028cd4875\n            correlation ID: 7e9c28a1-2d97-4241-837c-65dbd7be5ebd\n            client request ID: 76ea1b85-c87c-48c0-b8b1-76222a78e914\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"b136a9b186b73309fc330d90067542e4a0acbb1b86014bb556a7bc37a5f5c48c","pr_number":7248,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"de8a7215be9cbe7df7ff3c38f8eba50243fbc0204a90552d4bf737d8d09cb701","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingOnInfrastructureReady: Cluster infrastructure is still provisioning; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-w5zxw6n76pst/cluster-candidate-4-20-dnk6t4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x4d01c384420\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-dnk6t4: failed waiting for cluster=\"cluster-candidate-4-20-dnk6t4\" in resourcegroup=\"rg-candidate-4-20-w5zxw6n76pst\" to finish creating: GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\",\n      \"name\": \"d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T12:39:39.471658263Z\",\n      \"endTime\": \"2026-10-13T13:08:48.195755915Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingOnInfrastructureReady: Cluster infrastructure is still provisioning; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\n            correlation ID: 6fd24ae1-3485-4cf6-8f00-56a00457a1d3\n            client request ID: 52d0a7ef-5194-4884-9aba-b6475a9fa220\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-w5zxw6n76pst/cluster-candidate-4-20-dnk6t4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x4d01c384420\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-dnk6t4: failed waiting for cluster=\"cluster-candidate-4-20-dnk6t4\" in resourcegroup=\"rg-candidate-4-20-w5zxw6n76pst\" to finish creating: GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\",\n      \"name\": \"d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T12:39:39.471658263Z\",\n      \"endTime\": \"2026-10-13T13:08:48.195755915Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingOnInfrastructureReady: Cluster infrastructure is still provisioning; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\n            correlation ID: 6fd24ae1-3485-4cf6-8f00-56a00457a1d3\n            client request ID: 52d0a7ef-5194-4884-9aba-b6475a9fa220\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109972734464233472","occurred_at":"2026-10-13T11:41:15Z","signature_id":"c12ed8153f187823b5da8c9f2feb1648642f6798384985cc1969cf91f57dcead","pr_number":7485,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"27aef5ac99f567baefaad85029e0d3da9b4dbe40c6bbe0be985920a793730d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterValidation] DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0x495f5a50100\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-vtqpftbtl8pd\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\",\n      \"name\": \"b568c3f0-0cc4-47ab-8959-64f2a472fc67\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T16:05:12.565546726Z\",\n      \"endTime\": \"2026-10-12T16:34:19.299056423Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterValidation] DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\n            correlation ID: 2f0e8af3-4477-4dd1-b4a2-0bb21a3ecc71\n            client request ID: af43345c-178c-4f70-985a-42472759437b\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0x495f5a50100\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-vtqpftbtl8pd\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\",\n      \"name\": \"b568c3f0-0cc4-47ab-8959-64f2a472fc67\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T16:05:12.565546726Z\",\n      \"endTime\": \"2026-10-12T16:34:19.299056423Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterValidation] DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\n            correlation ID: 2f0e8af3-4477-4dd1-b4a2-0bb21a3ecc71\n            client request ID: af43345c-178c-4f70-985a-42472759437b\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"59da2ca63ee261235a31615de260be397d214b0f4f694b160b2ac972a17ed31e","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"12b9eebb9a2f4d0ab205bcc3542c32997c4cabea8b190cb208397a3ed523c5bb","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service is pending; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-8shnfp\nUnexpected error:\n    \u003c*fmt.wrapError | 0x272e47b96540\u003e: \n    failed to create HCP cluster np-noedge-8shnfp: failed waiting for cluster=\"np-noedge-8shnfp\" in resourcegroup=\"rg-np-noedge-8shnfp-gttkb9wtnnb4\" to finish creating: GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6eb1774a-462c-4933-9f8d-664c52f9a6f9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6eb1774a-462c-4933-9f8d-664c52f9a6f9\",\n      \"name\": \"6eb1774a-462c-4933-9f8d-664c52f9a6f9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T17:57:55.947947599Z\",\n      \"endTime\": \"2026-10-15T18:27:03.337200298Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service is pending\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6eb1774a-462c-4933-9f8d-664c52f9a6f9\n            correlation ID: aed6b19e-e002-4183-8925-f15bb6214fc7\n            client request ID: 45dff945-3cb8-4467-a1f3-78455ef2f77e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-8shnfp\nUnexpected error:\n    \u003c*fmt.wrapError | 0x272e47b96540\u003e: \n    failed to create HCP cluster np-noedge-8shnfp: failed waiting for cluster=\"np-noedge-8shnfp\" in resourcegroup=\"rg-np-noedge-8shnfp-gttkb9wtnnb4\" to finish creating: GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6eb1774a-462c-4933-9f8d-664c52f9a6f9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6eb1774a-462c-4933-9f8d-664c52f9a6f9\",\n      \"name\": \"6eb1774a-462c-4933-9f8d-664c52f9a6f9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T17:57:55.947947599Z\",\n      \"endTime\": \"2026-10-15T18:27:03.337200298Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service is pending\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3520000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6eb1774a-462c-4933-9f8d-664c52f9a6f9\n            correlation ID: aed6b19e-e002-4183-8925-f15bb6214fc7\n            client request ID: 45dff945-3cb8-4467-a1f3-78455ef2f77e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110772387883520000","occurred_at":"2026-10-15T16:38:43Z","signature_id":"14f6aed9cb73b38b2120958fe8cdfd834157065038eb89f35bd24e74c752b134","pr_number":7560,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b89623a31d2776290135b67a1c3e8137de86d30ee19f034e5531fb48ad00d00f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:135]: failed to create HCP cluster des-encrypt\nUnexpected error:\n    \u003c*fmt.wrapError | 0xe13d9ba2880\u003e: \n    failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-9czhr6hdgkds\" to finish creating: GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\",\n      \"name\": \"abdd1b6c-8284-4503-9367-9545ab4b7e3c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T18:45:22.752877991Z\",\n      \"endTime\": \"2026-10-12T19:14:30.175613398Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 20m44s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\n            correlation ID: e99f3924-5824-4d14-bfb3-a5c2c635c6fb\n            client request ID: 3e3fb9a9-7a4c-4e3b-9ae6-64fae2a5931a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:135]: failed to create HCP cluster des-encrypt\nUnexpected error:\n    \u003c*fmt.wrapError | 0xe13d9ba2880\u003e: \n    failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-9czhr6hdgkds\" to finish creating: GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\",\n      \"name\": \"abdd1b6c-8284-4503-9367-9545ab4b7e3c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T18:45:22.752877991Z\",\n      \"endTime\": \"2026-10-12T19:14:30.175613398Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 20m44s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\n            correlation ID: e99f3924-5824-4d14-bfb3-a5c2c635c6fb\n            client request ID: 3e3fb9a9-7a4c-4e3b-9ae6-64fae2a5931a\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109704412896169984","occurred_at":"2026-10-12T17:55:03Z","signature_id":"01fffc7b6758239c11fdf9bc2bb6322f5a22ddef4368d43104f752d3e34ce9e2","pr_number":7467,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4e831f602c4ea7f0cce0ce97a860291c59c5abae603b8e4d909a59a3f966062d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-\u003cid\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-\u003cid\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2e793540e048\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4b88pr2h6f62\" to finish creating: GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02548047-b3fd-49b8-8426-7df711bcbfbe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02548047-b3fd-49b8-8426-7df711bcbfbe\",\n      \"name\": \"02548047-b3fd-49b8-8426-7df711bcbfbe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T10:01:14.415831901Z\",\n      \"endTime\": \"2026-10-14T10:30:17.083278262Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-794ws-42z9w: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-42z9w to exist\\nMachine cilium-cluster-cilium-np-794ws-44qrb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-44qrb to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-794ws-42z9w: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-42z9w to exist\\nMachine cilium-cluster-cilium-np-794ws-44qrb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-44qrb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02548047-b3fd-49b8-8426-7df711bcbfbe\n            correlation ID: d40a80e8-ffc2-4aca-bd50-98d7806c38e8\n            client request ID: 84b1cf39-1ee3-4c6a-80ac-30f66febe153\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2e793540e048\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4b88pr2h6f62\" to finish creating: GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02548047-b3fd-49b8-8426-7df711bcbfbe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02548047-b3fd-49b8-8426-7df711bcbfbe\",\n      \"name\": \"02548047-b3fd-49b8-8426-7df711bcbfbe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T10:01:14.415831901Z\",\n      \"endTime\": \"2026-10-14T10:30:17.083278262Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-794ws-42z9w: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-42z9w to exist\\nMachine cilium-cluster-cilium-np-794ws-44qrb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-44qrb to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-794ws-42z9w: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-42z9w to exist\\nMachine cilium-cluster-cilium-np-794ws-44qrb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-44qrb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02548047-b3fd-49b8-8426-7df711bcbfbe\n            correlation ID: d40a80e8-ffc2-4aca-bd50-98d7806c38e8\n            client request ID: 84b1cf39-1ee3-4c6a-80ac-30f66febe153\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110291728471691264","occurred_at":"2026-10-14T08:48:48Z","signature_id":"d276781ea9d969dec523b2e2c4626568e236e9931f92b3fabae1fbe0cc6e76bf","pr_number":7500,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2a17ea6624b520ae513ca4725852ca83305c70a471a12e833ed2d7d18cfe8aaa","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-\u003cid\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-\u003cid\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x147375bd60d8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-l8lntszfbqhk\" to finish creating: GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a\",\n      \"name\": \"1bccabc3-2126-4728-9d49-4eea01285f0a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T01:45:21.225915122Z\",\n      \"endTime\": \"2026-10-14T02:14:22.961369975Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-llxzk-7ntxq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-7ntxq to exist\\nMachine cilium-cluster-cilium-np-llxzk-gsvtb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-llxzk-7ntxq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-7ntxq to exist\\nMachine cilium-cluster-cilium-np-llxzk-gsvtb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a\n            correlation ID: 854376b3-85ce-463a-aabe-5fc52fac910e\n            client request ID: d86cb5ff-fe28-499e-baef-054a0de770d9\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x147375bd60d8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-l8lntszfbqhk\" to finish creating: GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a\",\n      \"name\": \"1bccabc3-2126-4728-9d49-4eea01285f0a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T01:45:21.225915122Z\",\n      \"endTime\": \"2026-10-14T02:14:22.961369975Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-llxzk-7ntxq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-7ntxq to exist\\nMachine cilium-cluster-cilium-np-llxzk-gsvtb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-llxzk-7ntxq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-7ntxq to exist\\nMachine cilium-cluster-cilium-np-llxzk-gsvtb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a\n            correlation ID: 854376b3-85ce-463a-aabe-5fc52fac910e\n            client request ID: d86cb5ff-fe28-499e-baef-054a0de770d9\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110168566228586496","occurred_at":"2026-10-14T00:39:25Z","signature_id":"d1401fd1962c2b0a341d2d8996ab42716af9f4edd450ddb234863dd75ed0017e","pr_number":7248,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4429f82a5f862d3bc6430898431065e588f0c53e9df67037818ee8bac0354c92","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30c7dd043d80\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-bkhxjv72lv2h\" to finish creating: GET https://rp.j5369728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a2658dd-7908-49f0-96ce-e3959f8275fe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a2658dd-7908-49f0-96ce-e3959f8275fe\",\n      \"name\": \"4a2658dd-7908-49f0-96ce-e3959f8275fe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T11:58:55.879104397Z\",\n      \"endTime\": \"2026-10-15T12:27:56.020229321Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine des-encrypt-des-np-nv5pg-24w7d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-bkhxjv72lv2h--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-nv5pg-24w7d to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine des-encrypt-des-np-nv5pg-24w7d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-bkhxjv72lv2h--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-nv5pg-24w7d to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5369728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a2658dd-7908-49f0-96ce-e3959f8275fe\n            correlation ID: 89998081-e643-4b0b-aa81-6a2420c250ac\n            client request ID: 66cbea70-ac76-440c-ab1a-21754876dae9\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30c7dd043d80\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-bkhxjv72lv2h\" to finish creating: GET https://rp.j5369728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a2658dd-7908-49f0-96ce-e3959f8275fe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a2658dd-7908-49f0-96ce-e3959f8275fe\",\n      \"name\": \"4a2658dd-7908-49f0-96ce-e3959f8275fe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-15T11:58:55.879104397Z\",\n      \"endTime\": \"2026-10-15T12:27:56.020229321Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine des-encrypt-des-np-nv5pg-24w7d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-bkhxjv72lv2h--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-nv5pg-24w7d to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine des-encrypt-des-np-nv5pg-24w7d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-bkhxjv72lv2h--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-nv5pg-24w7d to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5369728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a2658dd-7908-49f0-96ce-e3959f8275fe\n            correlation ID: 89998081-e643-4b0b-aa81-6a2420c250ac\n            client request ID: 66cbea70-ac76-440c-ab1a-21754876dae9\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110683371255369728","occurred_at":"2026-10-15T10:44:49Z","signature_id":"4eb19ee505858631a103582454f2182a355a1ed4eba9d9dda97f8584aeaeefdf","pr_number":7553,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"9753620db9bca915ac646a67bd2e0c008b9f7be56f68db99d81d98334bd0e945","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-6rhdr-cwgtg to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-6rhdr-cwgtg to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x190aa1492400\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-qbnc288qgj6j\" to finish creating: GET https://rp.j3865088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199b8626-fe4e-4b24-9607-bad9bdfbcbe7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199b8626-fe4e-4b24-9607-bad9bdfbcbe7\",\n      \"name\": \"199b8626-fe4e-4b24-9607-bad9bdfbcbe7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T20:29:55.420741799Z\",\n      \"endTime\": \"2026-10-14T20:59:01.637394871Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-6rhdr-cwgtg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-qbnc288qgj6j--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-6rhdr-cwgtg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-6rhdr-cwgtg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-qbnc288qgj6j--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-6rhdr-cwgtg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3865088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199b8626-fe4e-4b24-9607-bad9bdfbcbe7\n            correlation ID: 1367eb49-5332-4cf5-9a4a-4b806be9f934\n            client request ID: 9a4866d1-15fd-4118-b349-e831018d7dac\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x190aa1492400\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-qbnc288qgj6j\" to finish creating: GET https://rp.j3865088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199b8626-fe4e-4b24-9607-bad9bdfbcbe7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199b8626-fe4e-4b24-9607-bad9bdfbcbe7\",\n      \"name\": \"199b8626-fe4e-4b24-9607-bad9bdfbcbe7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T20:29:55.420741799Z\",\n      \"endTime\": \"2026-10-14T20:59:01.637394871Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-6rhdr-cwgtg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-qbnc288qgj6j--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-6rhdr-cwgtg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-6rhdr-cwgtg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-qbnc288qgj6j--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-6rhdr-cwgtg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3865088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199b8626-fe4e-4b24-9607-bad9bdfbcbe7\n            correlation ID: 1367eb49-5332-4cf5-9a4a-4b806be9f934\n            client request ID: 9a4866d1-15fd-4118-b349-e831018d7dac\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452260763865088","occurred_at":"2026-10-14T19:26:43Z","signature_id":"be91c275aa99e6fe4f3186dff45127b70f53e8965e714209fe971eb77d5dcd19","pr_number":7519,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"08c1c1b279d699e6f9a43271a8c32e8779da05f6b9c675b49ef7772e07d5726d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-nzqkm-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-nzqkm-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a53ed53e100\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-b6z74shd5q4f\" to finish creating: GET https://rp.j0292224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e37b257b-a800-4d32-9117-6f5dadd22fa8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e37b257b-a800-4d32-9117-6f5dadd22fa8\",\n      \"name\": \"e37b257b-a800-4d32-9117-6f5dadd22fa8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T07:43:11.635376265Z\",\n      \"endTime\": \"2026-10-14T08:12:15.916382682Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-nzqkm-w8mpn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b6z74shd5q4f--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-nzqkm-w8mpn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-nzqkm-w8mpn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b6z74shd5q4f--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-nzqkm-w8mpn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0292224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e37b257b-a800-4d32-9117-6f5dadd22fa8\n            correlation ID: a458a580-f3b1-4361-8ade-4eb50ff689a4\n            client request ID: ac6a0de7-84cd-4baa-8bf7-d9a0e674cba5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a53ed53e100\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-b6z74shd5q4f\" to finish creating: GET https://rp.j0292224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e37b257b-a800-4d32-9117-6f5dadd22fa8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e37b257b-a800-4d32-9117-6f5dadd22fa8\",\n      \"name\": \"e37b257b-a800-4d32-9117-6f5dadd22fa8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T07:43:11.635376265Z\",\n      \"endTime\": \"2026-10-14T08:12:15.916382682Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-nzqkm-w8mpn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b6z74shd5q4f--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-nzqkm-w8mpn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-nzqkm-w8mpn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b6z74shd5q4f--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-nzqkm-w8mpn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0292224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e37b257b-a800-4d32-9117-6f5dadd22fa8\n            correlation ID: a458a580-f3b1-4361-8ade-4eb50ff689a4\n            client request ID: ac6a0de7-84cd-4baa-8bf7-d9a0e674cba5\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110259740570292224","occurred_at":"2026-10-14T06:41:43Z","signature_id":"fea509d928df2e88f833dc8d3fe2fd070af86fa3b0889acfe2996fef09b6fa2b","pr_number":7509,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f71af0d393321579ba6f1aaa05e8bc29e7a762e5437ac44b1d36d3dedbcacbe3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-rpntb-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-rpntb-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x346ef5b24080\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-xn2t4zqk2qxz\" to finish creating: GET https://rp.j6626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/439dbd7f-6ac7-4b48-8533-a8815df6b20f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/439dbd7f-6ac7-4b48-8533-a8815df6b20f\",\n      \"name\": \"439dbd7f-6ac7-4b48-8533-a8815df6b20f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T12:55:20.912229682Z\",\n      \"endTime\": \"2026-10-14T13:24:20.249835656Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-rpntb-stl8c: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-xn2t4zqk2qxz--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-rpntb-stl8c to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-rpntb-stl8c: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-xn2t4zqk2qxz--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-rpntb-stl8c to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/439dbd7f-6ac7-4b48-8533-a8815df6b20f\n            correlation ID: eefc14a4-21ec-420e-9b63-e8df580da42a\n            client request ID: b9d8fbaf-fe74-4488-8491-9a8ef5ba8fd7\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x346ef5b24080\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-xn2t4zqk2qxz\" to finish creating: GET https://rp.j6626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/439dbd7f-6ac7-4b48-8533-a8815df6b20f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/439dbd7f-6ac7-4b48-8533-a8815df6b20f\",\n      \"name\": \"439dbd7f-6ac7-4b48-8533-a8815df6b20f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T12:55:20.912229682Z\",\n      \"endTime\": \"2026-10-14T13:24:20.249835656Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-rpntb-stl8c: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-xn2t4zqk2qxz--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-rpntb-stl8c to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-rpntb-stl8c: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-xn2t4zqk2qxz--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-rpntb-stl8c to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/439dbd7f-6ac7-4b48-8533-a8815df6b20f\n            correlation ID: eefc14a4-21ec-420e-9b63-e8df580da42a\n            client request ID: b9d8fbaf-fe74-4488-8491-9a8ef5ba8fd7\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7525/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110338041116626944","occurred_at":"2026-10-14T11:52:51Z","signature_id":"e82a3bd52f8fdc258c91f1db11faa8c8e871c65a7359e7d18ab9e373fa688165","pr_number":7525,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"95af36169869353e6e4bd033f76bffdcb34f40de57dacc0d2314155a545063e2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-fqbnw to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-qvbld to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-fqbnw to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-qvbld to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xcf2616b6280\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-mfxbjq8jns8s\" to finish creating: GET https://rp.j9450880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ef2bbd7-ca39-48e4-90ab-26948b5b65f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ef2bbd7-ca39-48e4-90ab-26948b5b65f1\",\n      \"name\": \"6ef2bbd7-ca39-48e4-90ab-26948b5b65f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T07:49:18.1783953Z\",\n      \"endTime\": \"2026-10-14T08:18:20.880236175Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v20261001-np-1-cw6kh-fqbnw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-fqbnw to exist\\nMachine v20261001-np-1-cw6kh-qvbld: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-qvbld to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20261001-np-1-cw6kh-fqbnw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-fqbnw to exist\\nMachine v20261001-np-1-cw6kh-qvbld: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-qvbld to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9450880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ef2bbd7-ca39-48e4-90ab-26948b5b65f1\n            correlation ID: 39af48df-821f-41b4-9eda-383af9fbc767\n            client request ID: d214e485-a699-4f3c-bb9e-2d4e1497eed9\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xcf2616b6280\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-mfxbjq8jns8s\" to finish creating: GET https://rp.j9450880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ef2bbd7-ca39-48e4-90ab-26948b5b65f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ef2bbd7-ca39-48e4-90ab-26948b5b65f1\",\n      \"name\": \"6ef2bbd7-ca39-48e4-90ab-26948b5b65f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-14T07:49:18.1783953Z\",\n      \"endTime\": \"2026-10-14T08:18:20.880236175Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v20261001-np-1-cw6kh-fqbnw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-fqbnw to exist\\nMachine v20261001-np-1-cw6kh-qvbld: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-qvbld to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20261001-np-1-cw6kh-fqbnw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-fqbnw to exist\\nMachine v20261001-np-1-cw6kh-qvbld: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-qvbld to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9450880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ef2bbd7-ca39-48e4-90ab-26948b5b65f1\n            correlation ID: 39af48df-821f-41b4-9eda-383af9fbc767\n            client request ID: d214e485-a699-4f3c-bb9e-2d4e1497eed9\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110261282979450880","occurred_at":"2026-10-14T06:47:51Z","signature_id":"d62d439af2428ff5da7588c1c8cdf6f1dcb51f4f80f91fc98fcda9890d13348f","pr_number":6712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f957ba9aa3421e976868190e9cbb9bb30db37080a11af2efd3d47ffe3cc63196","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: NotReady: * NodeHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d04d19fc660\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-kjxkhz\" in resourcegroup=\"rg-np-version-upgrade-kjxkhz-ldkgrb6cvc64\" to finish creating: GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eec0983a-6d6f-417c-a350-df06c778abae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eec0983a-6d6f-417c-a350-df06c778abae\",\n      \"name\": \"eec0983a-6d6f-417c-a350-df06c778abae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T21:16:56.042187584Z\",\n      \"endTime\": \"2026-10-16T21:35:57.540237831Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o9t1p7r8v5x0r1j-npupgrade-4-20-jqtb8-n8bff: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o9t1p7r8v5x0r1j-npupgrade-4-20-jqtb8-n8bff: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eec0983a-6d6f-417c-a350-df06c778abae\n            correlation ID: 62969ebc-fdd1-4a1a-808f-bd02fcd158bd\n            client request ID: f0c87d09-298f-426d-885a-00fac76d2bf1\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d04d19fc660\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-kjxkhz\" in resourcegroup=\"rg-np-version-upgrade-kjxkhz-ldkgrb6cvc64\" to finish creating: GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eec0983a-6d6f-417c-a350-df06c778abae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eec0983a-6d6f-417c-a350-df06c778abae\",\n      \"name\": \"eec0983a-6d6f-417c-a350-df06c778abae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T21:16:56.042187584Z\",\n      \"endTime\": \"2026-10-16T21:35:57.540237831Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o9t1p7r8v5x0r1j-npupgrade-4-20-jqtb8-n8bff: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o9t1p7r8v5x0r1j-npupgrade-4-20-jqtb8-n8bff: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eec0983a-6d6f-417c-a350-df06c778abae\n            correlation ID: 62969ebc-fdd1-4a1a-808f-bd02fcd158bd\n            client request ID: f0c87d09-298f-426d-885a-00fac76d2bf1\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7531/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111186341810147328","occurred_at":"2026-10-16T20:03:41Z","signature_id":"05641d1f46b281713df86eeeae3a1d62541348ec25f684921bc566e261191e5e","pr_number":7531,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d82ea80612ae9d46dde4279310ffe224a3955ba4144d71e7257a94d2b7bd0e3c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1eddd58959c0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-brlqmmsvjbp7\" to finish creating: GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ec84baa-1f08-44e4-8884-c7d91686d7f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ec84baa-1f08-44e4-8884-c7d91686d7f5\",\n      \"name\": \"1ec84baa-1f08-44e4-8884-c7d91686d7f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T10:26:32.519187505Z\",\n      \"endTime\": \"2026-10-16T10:45:34.937263417Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-brlqmmsvjbp7--managed/providers/Microsoft.Compute/virtualMachines/f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-brlqmmsvjbp7--managed/providers/Microsoft.Compute/virtualMachines/f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ec84baa-1f08-44e4-8884-c7d91686d7f5\n            correlation ID: 6fd1b613-3112-4dbd-9dff-6f44c7554f29\n            client request ID: 0ac1b9cb-8173-4c03-9f5c-ab7b1db7762b\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1eddd58959c0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-brlqmmsvjbp7\" to finish creating: GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ec84baa-1f08-44e4-8884-c7d91686d7f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ec84baa-1f08-44e4-8884-c7d91686d7f5\",\n      \"name\": \"1ec84baa-1f08-44e4-8884-c7d91686d7f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-16T10:26:32.519187505Z\",\n      \"endTime\": \"2026-10-16T10:45:34.937263417Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-brlqmmsvjbp7--managed/providers/Microsoft.Compute/virtualMachines/f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-brlqmmsvjbp7--managed/providers/Microsoft.Compute/virtualMachines/f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ec84baa-1f08-44e4-8884-c7d91686d7f5\n            correlation ID: 6fd1b613-3112-4dbd-9dff-6f44c7554f29\n            client request ID: 0ac1b9cb-8173-4c03-9f5c-ab7b1db7762b\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7574/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111024523175268352","occurred_at":"2026-10-16T09:20:41Z","signature_id":"a18b5cfc4a13cbde1b572bc91b24cdf754420812d9b921eac4fc1bfe1226db6a","pr_number":7574,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"531a2084e8686f9a42755ac056f2dd31faa67fc9e5c5979d05857ee0dfb1b975","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentActive; detail message The deployment with resource id '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '\u003ctimestamp\u003e' with correlationId '\u003cuuid\u003e', and will expire at '\u003ctimestamp\u003e' if it does not complete before then.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:88]: failed to create customer resources for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd4464611460\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cidr-connectivity-test-b4xvfj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-b4xvfj/operationStatuses/08584097762288469100\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"DeploymentActive\\\",\\r\\n    \\\"message\\\": \\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-2hklr4tmfmv6/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '10/12/2026 6:44:17 PM' with correlationId '483786d1-9a79-473c-9660-1216b08e7f90', and will expire at '10/19/2026 6:44:17 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-b4xvfj/operationStatuses/08584097762288469100\n            correlation ID: d9670e5b-9049-4df2-b486-a7bed21f23b8\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:88]: failed to create customer resources for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd4464611460\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cidr-connectivity-test-b4xvfj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-b4xvfj/operationStatuses/08584097762288469100\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"DeploymentActive\\\",\\r\\n    \\\"message\\\": \\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-2hklr4tmfmv6/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '10/12/2026 6:44:17 PM' with correlationId '483786d1-9a79-473c-9660-1216b08e7f90', and will expire at '10/19/2026 6:44:17 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-b4xvfj/operationStatuses/08584097762288469100\n            correlation ID: d9670e5b-9049-4df2-b486-a7bed21f23b8\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109704412896169984","occurred_at":"2026-10-12T17:55:03Z","signature_id":"01033c5c618998eef4c756c29b116f1c2e1c20ad91b1f1d16200cebb174aafe7","pr_number":7467,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d7dfb9e704f567d65cc7ccb0bddcc27f123f1af0b24cca84fa1042281e3a756c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InvalidTemplate; detail message Unable to process template language expressions for resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Authorization/roleAssignments/\u003cuuid\u003e' at line '\u003cline\u003e' and column '\u003ccolumn\u003e'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3398cf888a80\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-autoscaling-hcp-cluster-qzrlv9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-qzrlv9/operationStatuses/08584097109520399682\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-cml7gjf827gp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-e4ixwyxsfjnje\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"InvalidTemplate\\\",\\r\\n            \\\"message\\\": \\\"Unable to process template language expressions for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-cml7gjf827gp/providers/Microsoft.Authorization/roleAssignments/adc91f81-bb61-5a2b-b4cc-0e8c1c460b92' at line '1' and column '27135'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'\\\",\\r\\n            \\\"additionalInfo\\\": [\\r\\n              {\\r\\n                \\\"type\\\": \\\"TemplateViolation\\\",\\r\\n                \\\"info\\\": {\\r\\n                  \\\"lineNumber\\\": 1,\\r\\n                  \\\"linePosition\\\": 27135,\\r\\n                  \\\"path\\\": \\\"\\\"\\r\\n                }\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-qzrlv9/operationStatuses/08584097109520399682\n            correlation ID: 9860797c-226a-442a-b3bd-c7b7c62b766b\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3398cf888a80\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-autoscaling-hcp-cluster-qzrlv9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-qzrlv9/operationStatuses/08584097109520399682\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-cml7gjf827gp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-e4ixwyxsfjnje\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"InvalidTemplate\\\",\\r\\n            \\\"message\\\": \\\"Unable to process template language expressions for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-cml7gjf827gp/providers/Microsoft.Authorization/roleAssignments/adc91f81-bb61-5a2b-b4cc-0e8c1c460b92' at line '1' and column '27135'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'\\\",\\r\\n            \\\"additionalInfo\\\": [\\r\\n              {\\r\\n                \\\"type\\\": \\\"TemplateViolation\\\",\\r\\n                \\\"info\\\": {\\r\\n                  \\\"lineNumber\\\": 1,\\r\\n                  \\\"linePosition\\\": 27135,\\r\\n                  \\\"path\\\": \\\"\\\"\\r\\n                }\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-qzrlv9/operationStatuses/08584097109520399682\n            correlation ID: 9860797c-226a-442a-b3bd-c7b7c62b766b\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109978799176159232","occurred_at":"2026-10-13T12:05:21Z","signature_id":"0f4249b820a1949de5b8be0a7d0f2992e157ed0020fa38fe47a583da35703e09","pr_number":6920,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,2,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":9,"prior_last_seen_at":"2026-10-01T19:26:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-23-5wrsns\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9f731ad0ea0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-23-5wrsns-td2dnf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-gn9flc7pzw5b\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-23-gn9flc7pzw5b/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-23-5wrsns-td2dnf/operationStatuses/08584096807961245220\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-mve2kajh3vwg6' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-23-gn9flc7pzw5b/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-23-5wrsns-td2dnf/operationStatuses/08584096807961245220\n            correlation ID: 547339b0-2310-46d9-9b61-b7c398526e80\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-23-5wrsns\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9f731ad0ea0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-23-5wrsns-td2dnf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-gn9flc7pzw5b\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-23-gn9flc7pzw5b/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-23-5wrsns-td2dnf/operationStatuses/08584096807961245220\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-mve2kajh3vwg6' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-23-gn9flc7pzw5b/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-23-5wrsns-td2dnf/operationStatuses/08584096807961245220\n            correlation ID: 547339b0-2310-46d9-9b61-b7c398526e80\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110098887174787072","occurred_at":"2026-10-13T19:57:04Z","signature_id":"c26a9d223de96e2bc27095ea2774785d4152120405397e9ea2ecd9e4d14ed85b","pr_number":7503,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"72716b475dfdb36ca6de5b4a7866bbc958e1921fb2c6ea0b3467c231190cd12c","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-28"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-30T01:28:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:293]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d44881448c0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-noedge-m2mjbc-x5mdss\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-m2mjbc-x5mdss\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: DeploymentNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DeploymentNotFound\",\n        \"message\": \"Deployment 'mi-np-noedge-m2mjbc-x5mdss' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-m2mjbc-x5mdss\n            correlation ID: fd414cb6-2644-4527-9130-ea3d146c0317\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:293]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d44881448c0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-noedge-m2mjbc-x5mdss\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-m2mjbc-x5mdss\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: DeploymentNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DeploymentNotFound\",\n        \"message\": \"Deployment 'mi-np-noedge-m2mjbc-x5mdss' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-m2mjbc-x5mdss\n            correlation ID: fd414cb6-2644-4527-9130-ea3d146c0317\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7527/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110636208349515776","occurred_at":"2026-10-15T07:37:37Z","signature_id":"6ed343f9009ba7f8aeadffda717209f81f41aa3e7b414093ed494f3db600781a","pr_number":7527,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"64bb2607db9a31af645f91fcf6ffbdb70d8bca63638eea03e861ce2387c49227","failure_pattern":"ERROR CODE: DeploymentNotFound; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentNotFound","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-09T21:41:16Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-14T12:33:58.591Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-10-14T12:35:01.493Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-14T12:33:58.60113061Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5008768-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5008768-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-14T12:34:00.818434525Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5008768-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5008768-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-14T12:34:01.565464014Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5008768-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5008768-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-10-14T12:35:01.489201259Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"collecting deployment operations\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5008768-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5008768-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"err\\\":{\\\"errorCode\\\":\\\"DeploymentNotFound\\\",\\\"statusCode\\\":404,\\\"errorMessage\\\":\\\"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5008768-mgmt-2/deployments/aks-cluster-create/operations\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 404: 404 Not Found\\\\nERROR CODE: DeploymentNotFound\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentNotFound\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Deployment 'aks-cluster-create' could not be found.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"}}\\n{\\\"time\\\":\\\"2026-10-14T12:35:01.489304211Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5008768-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5008768-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"err\\\":\\\"submitting deployment aks-cluster-create: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5008768-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 400: 400 Bad Request\\\\nERROR CODE: InvalidTemplateDeployment\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"InvalidTemplateDeployment\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The template deployment 'aks-cluster-create' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.ContainerService/managedClusters (2026-04-02-preview)' reported preflight validation errors. Tracking id is 'cd5c0f42-620b-41a9-b4be-4210990b303c'. See inner errors for details.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": [\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"ListResourceSKUsFailed\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"Preflight validation check for resource(s) for container service ci01-j5008768-mgmt-2 in resource group hcp-underlay-ci01-j5008768-mgmt-2 failed. Message: List resource SKUs in subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX failed. Details: \\\\\\\"\\\\n      }\\\\n    ]\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"1m2.888193483s\\\"}\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7522/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110340744895008768","occurred_at":"2026-10-14T12:03:18Z","signature_id":"9c183fffa613e8634fa50b7ecf08c849a5c8665852ba5fbee19eb46bb04baa54","pr_number":7522,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fe24c8b4b28d3d2b1937803a92704c44ec574f3d1c3b740ec6d97282a1e18d39","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-28","2026-10-05"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-08T07:10:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ec259084840\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-tv5qjjvw6mt5\" to finish creating: GET https://rp.j8141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\",\n      \"name\": \"639bbef1-ebbd-4367-896d-cec3c0263d16\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T15:42:56.655438798Z\",\n      \"endTime\": \"2026-10-12T15:43:02.11596193Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\n            correlation ID: 1ceff82d-31f8-4b03-ac69-9b0cd1c3b903\n            client request ID: 24f2f909-e5ab-41c2-8601-8e5d6aea83f8\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ec259084840\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-tv5qjjvw6mt5\" to finish creating: GET https://rp.j8141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\",\n      \"name\": \"639bbef1-ebbd-4367-896d-cec3c0263d16\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T15:42:56.655438798Z\",\n      \"endTime\": \"2026-10-12T15:43:02.11596193Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\n            correlation ID: 1ceff82d-31f8-4b03-ac69-9b0cd1c3b903\n            client request ID: 24f2f909-e5ab-41c2-8601-8e5d6aea83f8\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109655959428141056","occurred_at":"2026-10-12T14:42:30Z","signature_id":"0b00e0911be997c70989e7dd3678d4edc17e5aca2336e84d6fe6cf814fadeb28","pr_number":7440,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7b00d884b9121fd268447d271afcfe83e409041219994587880e368f74202f32","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x12402d3b2b10\u003e: \n    resource group \"rg-candidate-4-23-qt7g4z697d5b\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-ch4n9f\" in resourcegroup=\"rg-candidate-4-23-qt7g4z697d5b\" to finish deleting: GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27faaa2f-8540-458b-a125-3f3536544cd1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27faaa2f-8540-458b-a125-3f3536544cd1\",\n      \"name\": \"27faaa2f-8540-458b-a125-3f3536544cd1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T12:59:59.269440518Z\",\n      \"endTime\": \"2026-10-13T13:24:08.339110012Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tdtgm7u9ipde1epc2rt5pe43isp14m7 still exists (deletion dispatched at 2026-10-13T12:59:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27faaa2f-8540-458b-a125-3f3536544cd1\n            correlation ID: 365de814-fd9f-4b17-8525-a2096af1cada\n            client request ID: 166f0483-5054-4255-b484-02e488295af5\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x12402d3b2b10\u003e: \n    resource group \"rg-candidate-4-23-qt7g4z697d5b\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-ch4n9f\" in resourcegroup=\"rg-candidate-4-23-qt7g4z697d5b\" to finish deleting: GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27faaa2f-8540-458b-a125-3f3536544cd1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27faaa2f-8540-458b-a125-3f3536544cd1\",\n      \"name\": \"27faaa2f-8540-458b-a125-3f3536544cd1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T12:59:59.269440518Z\",\n      \"endTime\": \"2026-10-13T13:24:08.339110012Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tdtgm7u9ipde1epc2rt5pe43isp14m7 still exists (deletion dispatched at 2026-10-13T12:59:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27faaa2f-8540-458b-a125-3f3536544cd1\n            correlation ID: 365de814-fd9f-4b17-8525-a2096af1cada\n            client request ID: 166f0483-5054-4255-b484-02e488295af5\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109972734464233472","occurred_at":"2026-10-13T11:41:15Z","signature_id":"ed359cc4388f74eb3c414e5bd2ce69baff8941bc6e08595174f886722dcbacee","pr_number":7485,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"40f7a558a9f14e3ad84a0dc62150fb9e1dcec198cbfa1c00cbbb104cbc63f3a7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-16T08:35:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x1a1cb20b4078\u003e: \n    resource group \"feature-aggregation-vthdss5fr9gf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-vthdss5fr9gf\" to finish deleting: GET https://rp.j7859840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a499b952-55a7-42b0-9b5b-9062dcf2e20d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a499b952-55a7-42b0-9b5b-9062dcf2e20d\",\n      \"name\": \"a499b952-55a7-42b0-9b5b-9062dcf2e20d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T01:31:01.090497823Z\",\n      \"endTime\": \"2026-10-13T01:55:09.444868511Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7859840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a499b952-55a7-42b0-9b5b-9062dcf2e20d\n            correlation ID: 77f162b5-13b3-4f68-9936-971476bd0866\n            client request ID: 9306570d-d008-4fe7-9f11-7b82a031625e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x1a1cb20b4078\u003e: \n    resource group \"feature-aggregation-vthdss5fr9gf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-vthdss5fr9gf\" to finish deleting: GET https://rp.j7859840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a499b952-55a7-42b0-9b5b-9062dcf2e20d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a499b952-55a7-42b0-9b5b-9062dcf2e20d\",\n      \"name\": \"a499b952-55a7-42b0-9b5b-9062dcf2e20d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-13T01:31:01.090497823Z\",\n      \"endTime\": \"2026-10-13T01:55:09.444868511Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7859840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a499b952-55a7-42b0-9b5b-9062dcf2e20d\n            correlation ID: 77f162b5-13b3-4f68-9936-971476bd0866\n            client request ID: 9306570d-d008-4fe7-9f11-7b82a031625e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7484/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109796058077859840","occurred_at":"2026-10-12T23:59:12Z","signature_id":"640722fdcf6ffb0b7a84d5246e5db08c2b424ef7488f0c86ca3478e9b885ef75","pr_number":7484,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c42f860bbf4cb76b671dc080497cd48bd19935942783e87b46a282ef4f701461","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"engineering-workers\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-09T11:01:56Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n    \u003c*fmt.wrapError | 0x72d88398300\u003e: \n    failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2m7pgwml79qx\": PUT https://rp.j4532096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2m7pgwml79qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Too long: may not be more than 15 bytes\",\n            \"target\": \"id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n            \"target\": \"id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://rp.j4532096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2m7pgwml79qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n            correlation ID: 0548989b-5739-4134-ae92-44e0b94372c4\n            client request ID: eff3cdde-c6fa-4be6-bf0f-91fa006cb430\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n    \u003c*fmt.wrapError | 0x72d88398300\u003e: \n    failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2m7pgwml79qx\": PUT https://rp.j4532096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2m7pgwml79qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Too long: may not be more than 15 bytes\",\n            \"target\": \"id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n            \"target\": \"id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://rp.j4532096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2m7pgwml79qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n            correlation ID: 0548989b-5739-4134-ae92-44e0b94372c4\n            client request ID: eff3cdde-c6fa-4be6-bf0f-91fa006cb430\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109643872534532096","occurred_at":"2026-10-12T13:54:09Z","signature_id":"1f4d7a78c6e6f2b64bfcac7b4fc7dadaf9ee3793c7757ef0699936b0dd1911ad","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"955f2e0940a6e134701e99a0c3ef85d6b5374445ed9f53e16789c85a8fd4d2bc","failure_pattern":"ERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.","search_query":"ERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-03T12:07:56Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-13T20:10:59.938Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=2 description=\"Step swift-vnet\\n  Kind: Shell\\n  Command: ./swift-vnet.sh\\n\"\ntime=2026-10-13T20:11:00.569Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: Failed to load or parse file /tmp/azure-cli-config-533168303/azureProfile.json. It will be overridden by default settings.\\nERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110091971564408832","occurred_at":"2026-10-13T19:26:38Z","signature_id":"9ca8c9ef2c96a2d57799808a4c23411872eadb74fb13e040cc050d45a0ce4d60","pr_number":7502,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"73cc879e56f570416795adcce7e08e28387abc22a0432f0cca452338bd4aff84","failure_pattern":"Get \"\u003curl\u003e\": context deadline exceeded","search_query":"Get \"https://console-openshift-console.apps.aro.private-kas.0ju5.j8150016.hcp.osadev.cloud\": context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:225]: Timed out after 600.000s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:223 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.0ju5.j8150016.hcp.osadev.cloud\": context deadline exceeded\nUnexpected error:\n    \u003c*url.Error | 0x36a4b7cb55f0\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.0ju5.j8150016.hcp.osadev.cloud\": context deadline exceeded\n    {\n        Op: \"Get\",\n        URL: \"https://console-openshift-console.apps.aro.private-kas.0ju5.j8150016.hcp.osadev.cloud\",\n        Err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:225]: Timed out after 600.000s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:223 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.0ju5.j8150016.hcp.osadev.cloud\": context deadline exceeded\nUnexpected error:\n    \u003c*url.Error | 0x36a4b7cb55f0\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.0ju5.j8150016.hcp.osadev.cloud\": context deadline exceeded\n    {\n        Op: \"Get\",\n        URL: \"https://console-openshift-console.apps.aro.private-kas.0ju5.j8150016.hcp.osadev.cloud\",\n        Err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110882585688150016","occurred_at":"2026-10-15T23:56:41Z","signature_id":"04562b33405089c704d9907e2836dc161b8a982b6412c22ae0e1168dbde02ea2","pr_number":7232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c146a310e9f36bad2b2031b81dadd86ffc315b9e8d8f9b9e60f16f2e9c3f9acf","failure_pattern":"NS lookup for aro","search_query":"NS lookup for aro","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:394]: Timed out after 90.001s.\ndelegation and authoritative NS answers should remain consistent for at least 30 seconds before issuance\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x371ee5ea5c0\u003e: \n    NS lookup for aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud via ns4-07.azure-dns.info.: lookup aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud. on 172.30.0.10:53: dial udp: lookup ns4-07.azure-dns.info.: i/o timeout\n    {\n        msg: \"NS lookup for aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud via ns4-07.azure-dns.info.: lookup aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud. on 172.30.0.10:53: dial udp: lookup ns4-07.azure-dns.info.: i/o timeout\",\n        err: \u003c*net.DNSError | 0x371ee500500\u003e{\n            UnwrapErr: \u003c*net.OpError | 0x371ee5004b0\u003e{\n                Op: \"dial\",\n                Net: \"udp\",\n                Source: nil,\n                Addr: nil,\n                Err: \u003c*net.DNSError | 0x371ee500460\u003e{\n                    UnwrapErr: \u003c*net.timeoutError | 0x17059be0\u003e{},\n                    Err: \"i/o timeout\",\n                    Name: \"ns4-07.azure-dns.info.\",\n                    Server: \"\",\n                    IsTimeout: true,\n                    IsTemporary: true,\n                    IsNotFound: false,\n                },\n            },\n            Err: \"dial udp: lookup ns4-07.azure-dns.info.: i/o timeout\",\n            Name: \"aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud.\",\n            Server: \"172.30.0.10:53\",\n            IsTimeout: true,\n            IsTemporary: true,\n            IsNotFound: false,\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:394]: Timed out after 90.001s.\ndelegation and authoritative NS answers should remain consistent for at least 30 seconds before issuance\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x371ee5ea5c0\u003e: \n    NS lookup for aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud via ns4-07.azure-dns.info.: lookup aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud. on 172.30.0.10:53: dial udp: lookup ns4-07.azure-dns.info.: i/o timeout\n    {\n        msg: \"NS lookup for aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud via ns4-07.azure-dns.info.: lookup aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud. on 172.30.0.10:53: dial udp: lookup ns4-07.azure-dns.info.: i/o timeout\",\n        err: \u003c*net.DNSError | 0x371ee500500\u003e{\n            UnwrapErr: \u003c*net.OpError | 0x371ee5004b0\u003e{\n                Op: \"dial\",\n                Net: \"udp\",\n                Source: nil,\n                Addr: nil,\n                Err: \u003c*net.DNSError | 0x371ee500460\u003e{\n                    UnwrapErr: \u003c*net.timeoutError | 0x17059be0\u003e{},\n                    Err: \"i/o timeout\",\n                    Name: \"ns4-07.azure-dns.info.\",\n                    Server: \"\",\n                    IsTimeout: true,\n                    IsTemporary: true,\n                    IsNotFound: false,\n                },\n            },\n            Err: \"dial udp: lookup ns4-07.azure-dns.info.: i/o timeout\",\n            Name: \"aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud.\",\n            Server: \"172.30.0.10:53\",\n            IsTimeout: true,\n            IsTemporary: true,\n            IsNotFound: false,\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110328620604985344","occurred_at":"2026-10-14T11:15:25Z","signature_id":"5d8b5b70f0a9c1c8487aba6b0baf30179d57edeb456ee14fadec7eb5d56e4327","pr_number":7524,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"58fc1cbd5bf08c0e9cacc04febc66fa47a6ea8ac19d3acc5b10cdd7a31b22169","failure_pattern":"RevokeCredentialsAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-mjx8hf\" in resourcegroup=\"admin-credential-lifecycle-test-k6wptwmmb252\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:222]: failed to revoke admin credentials for cluster \"admin-cred-lifecycle-mjx8hf\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x1ab46da90510\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-mjx8hf\" in resourcegroup=\"admin-credential-lifecycle-test-k6wptwmmb252\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-mjx8hf\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-k6wptwmmb252\\\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x1ab46cac9c40\u003e{\n                s: \"timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:222]: failed to revoke admin credentials for cluster \"admin-cred-lifecycle-mjx8hf\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x1ab46da90510\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-mjx8hf\" in resourcegroup=\"admin-credential-lifecycle-test-k6wptwmmb252\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-mjx8hf\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-k6wptwmmb252\\\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x1ab46cac9c40\u003e{\n                s: \"timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111378042730319872","occurred_at":"2026-10-17T08:45:27Z","signature_id":"70abbc638303e2bdc13f44e1dfd8ef96ce569ae304e7f4344645cf766995fb90","pr_number":7250,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7da5fb7000522ac8f196f5fe9701e1ceef598ae9e4de9c38e95fe95ace38654e","failure_pattern":"VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff)","search_query":"VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:408]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x165441e2cf48\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x165441dab3a0\u003e{\n                msg: \"VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]\",\n                err: \u003c*fmt.wrapError | 0x165441dab1c0\u003e{\n                    msg: \"connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]\",\n                    err: \u003c*errors.errorString | 0x1654427290b0\u003e{\n                        s: \"test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]\",\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:408]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x165441e2cf48\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x165441dab3a0\u003e{\n                msg: \"VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]\",\n                err: \u003c*fmt.wrapError | 0x165441dab1c0\u003e{\n                    msg: \"connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]\",\n                    err: \u003c*errors.errorString | 0x1654427290b0\u003e{\n                        s: \"test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]\",\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7531/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111218975433035776","occurred_at":"2026-10-16T22:13:22Z","signature_id":"232f7dd2673899fec5aeff9706e5509b4be07bc67cbc3af52825b2dc89f2cb5e","pr_number":7531,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"35f4b32d0c9a217f6e6dcfd9cb816a96a3bf1d26a1497dfd78708f5764de5a24","failure_pattern":"VerifyCloudNetworkConfigEgressIP(\u003cip\u003e) failed: EgressIP \"e2e-cloud-network-config-test\" was never assigned to a node: timed out waiting for EgressIP \"e2e-cloud-network-config-test\" to be assigned to a node: EgressIP \"e2e-cloud-network-config-test\" is not yet","search_query":"VerifyCloudNetworkConfigEgressIP(10.0.0.6) failed: EgressIP \"e2e-cloud-network-config-test\" was never assigned to a node: timed out waiting for EgressIP \"e2e-cloud-network-config-test\" to be assigned to a node: EgressIP \"e2e-cloud-network-config-test\" is not yet assigned to a node","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Verify cluster operators should provision an HCP cluster and verify the cluster operators","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Verify cluster operators should provision an HCP cluster and verify the cluster operators","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_operators.go:163]: failed to verify cluster operators on cluster \"cluster-operators-qwqhtt\"\nUnexpected error:\n    \u003c*errors.joinError | 0x275bb568108\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.6) failed: EgressIP \"e2e-cloud-network-config-test\" was never assigned to a node: timed out waiting for EgressIP \"e2e-cloud-network-config-test\" to be assigned to a node: EgressIP \"e2e-cloud-network-config-test\" is not yet assigned to a node\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x275bb4fe480\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.6) failed: EgressIP \\\"e2e-cloud-network-config-test\\\" was never assigned to a node: timed out waiting for EgressIP \\\"e2e-cloud-network-config-test\\\" to be assigned to a node: EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n                err: \u003c*fmt.wrapError | 0x275baa78f20\u003e{\n                    msg: \"EgressIP \\\"e2e-cloud-network-config-test\\\" was never assigned to a node: timed out waiting for EgressIP \\\"e2e-cloud-network-config-test\\\" to be assigned to a node: EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n                    err: \u003c*fmt.wrapError | 0x275baa78d80\u003e{\n                        msg: \"timed out waiting for EgressIP \\\"e2e-cloud-network-config-test\\\" to be assigned to a node: EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n                        err: \u003c*errors.errorString | 0x275bb7584c0\u003e{\n                            s: \"EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_operators.go:163]: failed to verify cluster operators on cluster \"cluster-operators-qwqhtt\"\nUnexpected error:\n    \u003c*errors.joinError | 0x275bb568108\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.6) failed: EgressIP \"e2e-cloud-network-config-test\" was never assigned to a node: timed out waiting for EgressIP \"e2e-cloud-network-config-test\" to be assigned to a node: EgressIP \"e2e-cloud-network-config-test\" is not yet assigned to a node\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x275bb4fe480\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.6) failed: EgressIP \\\"e2e-cloud-network-config-test\\\" was never assigned to a node: timed out waiting for EgressIP \\\"e2e-cloud-network-config-test\\\" to be assigned to a node: EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n                err: \u003c*fmt.wrapError | 0x275baa78f20\u003e{\n                    msg: \"EgressIP \\\"e2e-cloud-network-config-test\\\" was never assigned to a node: timed out waiting for EgressIP \\\"e2e-cloud-network-config-test\\\" to be assigned to a node: EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n                    err: \u003c*fmt.wrapError | 0x275baa78d80\u003e{\n                        msg: \"timed out waiting for EgressIP \\\"e2e-cloud-network-config-test\\\" to be assigned to a node: EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n                        err: \u003c*errors.errorString | 0x275bb7584c0\u003e{\n                            s: \"EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109977127079120896","occurred_at":"2026-10-13T11:58:43Z","signature_id":"bd20bcea67709cdf0163960d867a7fb6af562bc860fdce2d4fddc185e5ee2f10","pr_number":7463,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0a9bcf49d179171c63dccafd8dbb4e653af27553fa78df236def98914f6ada90","failure_pattern":"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xb019e0de2e8\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n    VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xb019c86fe00\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0xb019c86fda0\u003e{\n                    msg: \"failed to list services: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0xb019df811d0\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\",\n                        Err: \u003c*net.OpError | 0xb019c4d6ff0\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0xb019bfab1e0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0xb019bfab0e0\u003e{\n                    msg: \"failed to list all APIServices: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0xb019df140c0\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\",\n                        Err: \u003c*net.OpError | 0xb019c5e2a50\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0xb019c4e0980\u003e{\n                msg: \"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0xb019c4e0960\u003e{\n                    msg: \"failed to get default IngressController: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0xb019e2caff0\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\",\n                        Err: \u003c*net.OpError | 0xb019bfc47d0\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xb019e0de2e8\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n    VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xb019c86fe00\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0xb019c86fda0\u003e{\n                    msg: \"failed to list services: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0xb019df811d0\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\",\n                        Err: \u003c*net.OpError | 0xb019c4d6ff0\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0xb019bfab1e0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0xb019bfab0e0\u003e{\n                    msg: \"failed to list all APIServices: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0xb019df140c0\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\",\n                        Err: \u003c*net.OpError | 0xb019c5e2a50\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0xb019c4e0980\u003e{\n                msg: \"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0xb019c4e0960\u003e{\n                    msg: \"failed to get default IngressController: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0xb019e2caff0\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\",\n                        Err: \u003c*net.OpError | 0xb019bfc47d0\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109675831700754432","occurred_at":"2026-10-12T16:01:28Z","signature_id":"a87f271596a6f4e182a266365db74092bd67d3130b10c9e295b2604bd480ee54","pr_number":7162,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":15,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":46,"prior_last_seen_at":"2026-10-02T20:20:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0x11d59617f5d8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \"4.21\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x11d596138740\u003e{\n                msg: \"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\",\n                err: \u003c*errors.errorString | 0x11d59546a3c0\u003e{\n                    s: \"kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\",\n                },\n            },\n            \u003c*fmt.wrapError | 0x11d5961388a0\u003e{\n                msg: \"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \\\"4.21\\\"\",\n                err: \u003c*errors.errorString | 0x11d59546a6c0\u003e{\n                    s: \"clusterversion status.history has no version in target minor \\\"4.21\\\"\",\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0x11d59617f5d8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \"4.21\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x11d596138740\u003e{\n                msg: \"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\",\n                err: \u003c*errors.errorString | 0x11d59546a3c0\u003e{\n                    s: \"kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\",\n                },\n            },\n            \u003c*fmt.wrapError | 0x11d5961388a0\u003e{\n                msg: \"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \\\"4.21\\\"\",\n                err: \u003c*errors.errorString | 0x11d59546a6c0\u003e{\n                    s: \"clusterversion status.history has no version in target minor \\\"4.21\\\"\",\n                },\n            },\n        ],\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"e72a8b419adb6b53b3d96517db9d967450746716b4acc806c6d35e614fc4cc81","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7a87df562eaa8a0bfda8c14840c36e7197e35a3b92f22e4f536b1e3d2ed289ff","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-09-28"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-02T12:46:50Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x18c85c0bedc8\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x18c85ad32dc0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x18c85bf9a260\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x18c85bf9a240\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x18c85c0bedc8\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x18c85ad32dc0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x18c85bf9a260\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x18c85bf9a240\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rd295.apps.aro.cilium-cluster.c58v.j7178624.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7487/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110607153617178624","occurred_at":"2026-10-15T05:42:13Z","signature_id":"81b57b07ae09640e6ee1ce4f8b6361b066bdec840190e2be0ddc6c4048104564","pr_number":7487,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8e7e1939a67145fe288ccb305762ce0b966b7e2b9d52d9aa38f5a043a3e3e9f2","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: SWIFT query row","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T00:18:45.8286809Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\", ActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", SubActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityStack:\"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f \u003e GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\"}, IsPermanent:false})","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f63e25da900\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T00:18:45.8286809Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\", ActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", SubActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityStack:\"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f \u003e GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T00:18:45.8286809Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\\\", ActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", SubActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f \u003e GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x1f63e28703a0\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T00:18:45.8286809Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\\\", ActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", SubActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f \u003e GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x1f63e2d64800\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-15T00:18:45.8286809Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\",\n                        ActivityId: \"3e4bf140-f035-443b-96b4-bed0d927cd4f\",\n                        SubActivityId: \"3e4bf140-f035-443b-96b4-bed0d927cd4f\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"3e4bf140-f035-443b-96b4-bed0d927cd4f\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f \u003e GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f63e25da900\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T00:18:45.8286809Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\", ActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", SubActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityStack:\"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f \u003e GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T00:18:45.8286809Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\\\", ActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", SubActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f \u003e GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x1f63e28703a0\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T00:18:45.8286809Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\\\", ActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", SubActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"3e4bf140-f035-443b-96b4-bed0d927cd4f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f \u003e GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x1f63e2d64800\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-15T00:18:45.8286809Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\",\n                        ActivityId: \"3e4bf140-f035-443b-96b4-bed0d927cd4f\",\n                        SubActivityId: \"3e4bf140-f035-443b-96b4-bed0d927cd4f\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"3e4bf140-f035-443b-96b4-bed0d927cd4f\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f \u003e GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110505977575903232","occurred_at":"2026-10-14T23:00:10Z","signature_id":"dd27870673151fbc796c3edce7816006c004f55fff53275326afd3ff6f2ddb19","pr_number":7451,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"69e9a625fc3574b27da6a100519b2f81973de7aebd2a1ccd425027aa08783b72","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j1046272-mgmt-1/ocm-arohcpci01-\u003cid\u003e","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v9j4v95cl9wx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xadaa6040100\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v9j4v95cl9wx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j1046272-mgmt-1/ocm-arohcpci01-2th7q5a54cq5jldof0f3l7iofpg7u8al-h1p2a3c0s8o4u6j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v9j4v95cl9wx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j1046272-mgmt-1/ocm-arohcpci01-2th7q5a54cq5jldof0f3l7iofpg7u8al-h1p2a3c0s8o4u6j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0xadaa51438f0\u003e{\n            s: \"dev/canadacentral/ci01-j1046272-mgmt-1/ocm-arohcpci01-2th7q5a54cq5jldof0f3l7iofpg7u8al-h1p2a3c0s8o4u6j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v9j4v95cl9wx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xadaa6040100\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v9j4v95cl9wx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j1046272-mgmt-1/ocm-arohcpci01-2th7q5a54cq5jldof0f3l7iofpg7u8al-h1p2a3c0s8o4u6j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v9j4v95cl9wx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j1046272-mgmt-1/ocm-arohcpci01-2th7q5a54cq5jldof0f3l7iofpg7u8al-h1p2a3c0s8o4u6j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0xadaa51438f0\u003e{\n            s: \"dev/canadacentral/ci01-j1046272-mgmt-1/ocm-arohcpci01-2th7q5a54cq5jldof0f3l7iofpg7u8al-h1p2a3c0s8o4u6j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111797770821046272","occurred_at":"2026-10-18T12:33:18Z","signature_id":"b43432c9ea207ac983c242c108e787e684ab34e3936a86b9df05974662681ba9","pr_number":7591,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"91847fc55e97ea83d42ed15b0d83f685c1e0a138fec204f0e908e2bf134a6478","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jv8jpqmq9gpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x239a48fa84c0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jv8jpqmq9gpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-2tft4ek4kogcffngbs1nats12glqcn7n-x6a3m7l5u9v6a9d: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jv8jpqmq9gpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-2tft4ek4kogcffngbs1nats12glqcn7n-x6a3m7l5u9v6a9d: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0x239a4a108640\u003e{\n            s: \"dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-2tft4ek4kogcffngbs1nats12glqcn7n-x6a3m7l5u9v6a9d: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jv8jpqmq9gpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x239a48fa84c0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jv8jpqmq9gpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-2tft4ek4kogcffngbs1nats12glqcn7n-x6a3m7l5u9v6a9d: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jv8jpqmq9gpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-2tft4ek4kogcffngbs1nats12glqcn7n-x6a3m7l5u9v6a9d: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0x239a4a108640\u003e{\n            s: \"dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-2tft4ek4kogcffngbs1nats12glqcn7n-x6a3m7l5u9v6a9d: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7489/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111067237912678400","occurred_at":"2026-10-16T12:10:25Z","signature_id":"504867c83cb57dcb50cb4595712ff6bc2c40f73af164bfc8cf71f5840b752867","pr_number":7489,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4291ee8f6243a25b8eeb49236e9e84795ced38f439b9996bfab09d4200eb4d6c","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in \u003curl\u003e context deadline exceeded","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2620bace6480\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2620bace6460\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2620bace6480\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2620bace6460\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7525/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110338041116626944","occurred_at":"2026-10-14T11:52:51Z","signature_id":"e02e171a721a96c3dc0dd8058864c72da7053641616de4d9a79e76ea8f3ae577","pr_number":7525,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"45d0345406d5a551963c23a5ded3db1272e39d63a6c7ca06460fb46b276ed104","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lq67n7wxjv9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xa4868a84700\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lq67n7wxjv9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-2teu677bb6kl3jnq3h6qj1u9ribq025b-z3n3z0e0b6i7l6i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lq67n7wxjv9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-2teu677bb6kl3jnq3h6qj1u9ribq025b-z3n3z0e0b6i7l6i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0xa4869d391c0\u003e{\n            s: \"dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-2teu677bb6kl3jnq3h6qj1u9ribq025b-z3n3z0e0b6i7l6i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lq67n7wxjv9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xa4868a84700\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lq67n7wxjv9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-2teu677bb6kl3jnq3h6qj1u9ribq025b-z3n3z0e0b6i7l6i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lq67n7wxjv9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-2teu677bb6kl3jnq3h6qj1u9ribq025b-z3n3z0e0b6i7l6i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0xa4869d391c0\u003e{\n            s: \"dev/centralus/ci01-j9355392-mgmt-1/ocm-arohcpci01-2teu677bb6kl3jnq3h6qj1u9ribq025b-z3n3z0e0b6i7l6i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7540/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110540376279355392","occurred_at":"2026-10-15T01:16:52Z","signature_id":"afc5dc16004fe00ec4fadfcd21a3566176869acceef40b347c6fdeba6cb70b36","pr_number":7540,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1fb02875481a82a322575aa62ae21fd2ca87b013af435787776c9187994426c8","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tzcs2w52g775/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x6f640d28320\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tzcs2w52g775/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-2tf0946f9b41h1n7hltqnq95f64jvhk8-j9h5p7q3y7t6s2u: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tzcs2w52g775/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-2tf0946f9b41h1n7hltqnq95f64jvhk8-j9h5p7q3y7t6s2u: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x6f64075f4f0\u003e{\n            s: \"dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-2tf0946f9b41h1n7hltqnq95f64jvhk8-j9h5p7q3y7t6s2u: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tzcs2w52g775/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x6f640d28320\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tzcs2w52g775/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-2tf0946f9b41h1n7hltqnq95f64jvhk8-j9h5p7q3y7t6s2u: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tzcs2w52g775/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-2tf0946f9b41h1n7hltqnq95f64jvhk8-j9h5p7q3y7t6s2u: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x6f64075f4f0\u003e{\n            s: \"dev/westus3/ci01-j3692672-mgmt-2/ocm-arohcpci01-2tf0946f9b41h1n7hltqnq95f64jvhk8-j9h5p7q3y7t6s2u: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7401/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110572658033692672","occurred_at":"2026-10-15T03:25:08Z","signature_id":"079d9e11ecf4c9bf783efc15c5463fb2619c071c1ffcb6b29c1c7dce8abd75ae","pr_number":7401,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"6f5b0699b6f1437b23a1c7da6f46df6d98e827812483320febb57823f35e33eb","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in \u003curl\u003e context deadline exceeded","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3527505f6640\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3527505f6620\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3527505f6640\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jn4vtkkhs7mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3527505f6620\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110558398402531328","occurred_at":"2026-10-15T02:28:29Z","signature_id":"7361959de99cf2e3597ad30ee7670a2daf3f64f017ca0d0ae2cd989364d45570","pr_number":7497,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0d1bb767dc6bcd521c34a14e9b0517517350a11dc10d0bf7b970c9a0d6ce4727","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: SWIFT query row","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:29.6322749Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\", ActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", SubActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd \u003e GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\"}, IsPermanent:false})","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x21de3cbea400\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:29.6322749Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\", ActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", SubActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd \u003e GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T11:13:29.6322749Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\\\", ActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", SubActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd \u003e GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x21de3ca9e560\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T11:13:29.6322749Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\\\", ActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", SubActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd \u003e GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x21de3e2c5500\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-14T11:13:29.6322749Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\",\n                        ActivityId: \"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\",\n                        SubActivityId: \"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd \u003e GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x21de3cbea400\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:29.6322749Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\", ActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", SubActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd \u003e GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T11:13:29.6322749Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\\\", ActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", SubActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd \u003e GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x21de3ca9e560\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T11:13:29.6322749Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\\\", ActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", SubActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd \u003e GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x21de3e2c5500\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-14T11:13:29.6322749Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\",\n                        ActivityId: \"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\",\n                        SubActivityId: \"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd \u003e GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7522/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110302904370860032","occurred_at":"2026-10-14T09:30:21Z","signature_id":"28ee968824b537cbd60af98cfb420def9e8a07138c1b26600fb5f40c37a6e2be","pr_number":7522,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"01a075ff256ceb0e0b1d938a88d093e2a699f17fc729f7caecc4c334022cc991","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4hk6sg2whd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x21a95a4b0000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4hk6sg2whd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-2tfp6vu1t0esj9s2f87m52kogvfeb12i-d8h3a4b2v3x4x1h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4hk6sg2whd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-2tfp6vu1t0esj9s2f87m52kogvfeb12i-d8h3a4b2v3x4x1h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x21a95b98e500\u003e{\n            s: \"dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-2tfp6vu1t0esj9s2f87m52kogvfeb12i-d8h3a4b2v3x4x1h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4hk6sg2whd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x21a95a4b0000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4hk6sg2whd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-2tfp6vu1t0esj9s2f87m52kogvfeb12i-d8h3a4b2v3x4x1h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4hk6sg2whd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-2tfp6vu1t0esj9s2f87m52kogvfeb12i-d8h3a4b2v3x4x1h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x21a95b98e500\u003e{\n            s: \"dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-2tfp6vu1t0esj9s2f87m52kogvfeb12i-d8h3a4b2v3x4x1h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110999442776657920","occurred_at":"2026-10-16T07:41:02Z","signature_id":"bc274b8e050d4e114121e021987f8aef108de68d75ca046fc0ac61305af93eda","pr_number":7566,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4bdce0fc0263ab649c57301068b1470e004ac35ea6a6dc4e6dec1a88f568ae7f","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in \u003curl\u003e context deadline exceeded","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xee59296c020\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0xee59296c000\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xee59296c020\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0xee59296c000\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7523/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110300965679665152","occurred_at":"2026-10-14T09:25:32Z","signature_id":"2b1ef112e3fe566ad64f97a08cb37e00dd8e121aa32253ddb9ce1f4e565230f3","pr_number":7523,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fa2af3baaf9b229c16ea2eedc12bed8c9ba9d5ba824e8a6c12d3ccdfa46d6b6d","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lc82txzvgr45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x11ea08e84580\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lc82txzvgr45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-2tfouap5pj715l32pn7smi3ihfg5rglm-y5e3f0u2q1z1x6w: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lc82txzvgr45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-2tfouap5pj715l32pn7smi3ihfg5rglm-y5e3f0u2q1z1x6w: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0x11ea09d18c20\u003e{\n            s: \"dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-2tfouap5pj715l32pn7smi3ihfg5rglm-y5e3f0u2q1z1x6w: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lc82txzvgr45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x11ea08e84580\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lc82txzvgr45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-2tfouap5pj715l32pn7smi3ihfg5rglm-y5e3f0u2q1z1x6w: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lc82txzvgr45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-2tfouap5pj715l32pn7smi3ihfg5rglm-y5e3f0u2q1z1x6w: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0x11ea09d18c20\u003e{\n            s: \"dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-2tfouap5pj715l32pn7smi3ihfg5rglm-y5e3f0u2q1z1x6w: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110993516539678720","occurred_at":"2026-10-16T07:17:29Z","signature_id":"0f446af22185bf42597db517e6b3a284b2a5b5db844a92845ed8be29a77818cb","pr_number":7440,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5529031a8c5e7aeec420f57c88f8725911bfa302f5c294049d97b239d627c655","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-\u003cid\u003e","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cdzxpnm9wgtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3222464a4800\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cdzxpnm9wgtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-2tg0gb0ik8kqufv9aj7d0iha95qjnf3j-e7g2r4n0l4l5r1h router=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000: expected matching summary after test start and within 8m; summaryUID=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000 pass=2b7ade69-c160-46d5-8524-479bdeb90f91 at=2026-10-16T17:09:43Z\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cdzxpnm9wgtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-2tg0gb0ik8kqufv9aj7d0iha95qjnf3j-e7g2r4n0l4l5r1h router=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000: expected matching summary after test start and within 8m; summaryUID=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000 pass=2b7ade69-c160-46d5-8524-479bdeb90f91 at=2026-10-16T17:09:43Z\",\n        err: \u003c*errors.errorString | 0x322245a3b8b0\u003e{\n            s: \"dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-2tg0gb0ik8kqufv9aj7d0iha95qjnf3j-e7g2r4n0l4l5r1h router=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000: expected matching summary after test start and within 8m; summaryUID=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000 pass=2b7ade69-c160-46d5-8524-479bdeb90f91 at=2026-10-16T17:09:43Z\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cdzxpnm9wgtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3222464a4800\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cdzxpnm9wgtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-2tg0gb0ik8kqufv9aj7d0iha95qjnf3j-e7g2r4n0l4l5r1h router=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000: expected matching summary after test start and within 8m; summaryUID=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000 pass=2b7ade69-c160-46d5-8524-479bdeb90f91 at=2026-10-16T17:09:43Z\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cdzxpnm9wgtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-2tg0gb0ik8kqufv9aj7d0iha95qjnf3j-e7g2r4n0l4l5r1h router=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000: expected matching summary after test start and within 8m; summaryUID=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000 pass=2b7ade69-c160-46d5-8524-479bdeb90f91 at=2026-10-16T17:09:43Z\",\n        err: \u003c*errors.errorString | 0x322245a3b8b0\u003e{\n            s: \"dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-2tg0gb0ik8kqufv9aj7d0iha95qjnf3j-e7g2r4n0l4l5r1h router=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000: expected matching summary after test start and within 8m; summaryUID=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000 pass=2b7ade69-c160-46d5-8524-479bdeb90f91 at=2026-10-16T17:09:43Z\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111120828451000320","occurred_at":"2026-10-16T15:43:22Z","signature_id":"ab2ea88c29cdd316e2d7459a55135ddbe050949b4510bacb15d8ebb39608eed7","pr_number":6928,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ace3655e8c7da2e020456d9d99c4eaa589f6c874efd0e56cdc422ea75791aa32","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in \u003curl\u003e context deadline exceeded","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ba1a496c2e0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2ba1a496c2c0\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ba1a496c2e0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2ba1a496c2c0\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452244770983936","occurred_at":"2026-10-14T19:26:40Z","signature_id":"b0b4717112a8ce5606c552fa6abfbc35d74e436e05d5aea424433af3362f5a86","pr_number":7509,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c9083ec9c855a1c2a34a116ed4f451e20e6042d364acfc229b5b5f3aa5fba483","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in \u003curl\u003e context deadline exceeded","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x298032100120\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x298032100100\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x298032100120\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x298032100100\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7499/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452208637054976","occurred_at":"2026-10-14T19:26:26Z","signature_id":"d6ddfa46d3d745e923af613cb97a80b523ed08d2ae0ec6f2e9ddf174c80c66c2","pr_number":7499,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d44d2c4fe704ed3c143d81a48596157d54af9a75f3ee6b16e677749a6ef178d6","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: SWIFT query row","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:43.5550944Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\", ActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", SubActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityStack:\"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 \u003e GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\"}, IsPermanent:false})","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1dded76126c0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:43.5550944Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\", ActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", SubActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityStack:\"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 \u003e GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T11:13:43.5550944Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\\\", ActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", SubActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 \u003e GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x1dded76126a0\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T11:13:43.5550944Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\\\", ActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", SubActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 \u003e GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x1dded776cc00\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-14T11:13:43.5550944Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\",\n                        ActivityId: \"0eb2baf5-e792-455b-9c86-a0654265ab17\",\n                        SubActivityId: \"0eb2baf5-e792-455b-9c86-a0654265ab17\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"0eb2baf5-e792-455b-9c86-a0654265ab17\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 \u003e GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1dded76126c0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:43.5550944Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\", ActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", SubActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityStack:\"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 \u003e GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\"}, IsPermanent:false})\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T11:13:43.5550944Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\\\", ActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", SubActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 \u003e GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\\\"}, IsPermanent:false})\",\n        err: \u003c*fmt.wrapError | 0x1dded76126a0\u003e{\n            msg: \"SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T11:13:43.5550944Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\\\", ActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", SubActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0eb2baf5-e792-455b-9c86-a0654265ab17\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 \u003e GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\\\"}, IsPermanent:false})\",\n            err: \u003c*v2.OneApiError | 0x1dded776cc00\u003e{\n                ErrorMessage: {\n                    Code: \"LimitsExceeded\",\n                    Message: \"Request is invalid and cannot be executed.\",\n                    Description: \"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\",\n                    Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                    Context: {\n                        Timestamp: \"2026-10-14T11:13:43.5550944Z\",\n                        ServiceAlias: \"HCP-DEV-US-2\",\n                        MachineName: \"\",\n                        ProcessName: \"\",\n                        ProcessId: 0,\n                        ThreadId: 0,\n                        ClientRequestId: \"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\",\n                        ActivityId: \"0eb2baf5-e792-455b-9c86-a0654265ab17\",\n                        SubActivityId: \"0eb2baf5-e792-455b-9c86-a0654265ab17\",\n                        ActivityType: \"GW.Http.CallContext\",\n                        ParentActivityId: \"0eb2baf5-e792-455b-9c86-a0654265ab17\",\n                        ActivityStack: \"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 \u003e GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\",\n                    },\n                    IsPermanent: false,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110303886584582144","occurred_at":"2026-10-14T09:37:08Z","signature_id":"2f0e6a655ab2ba2b3047dea03f9db87c620221457bccc3eb57eb415eeee14c00","pr_number":7383,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ea70ac57f533e4db1817eb8e10ca7642713fd143538808aed9ae83504fe78bfe","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9276160-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-94d2g5wkbdwn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3b3acad40780\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-94d2g5wkbdwn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9276160-mgmt-1/ocm-arohcpci01-2tf7qg3qc9tr83rniscs197k8bo2fac2-o2o1a7a4r7x5y6r: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-94d2g5wkbdwn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9276160-mgmt-1/ocm-arohcpci01-2tf7qg3qc9tr83rniscs197k8bo2fac2-o2o1a7a4r7x5y6r: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x3b3acbd1bd90\u003e{\n            s: \"dev/westus3/ci01-j9276160-mgmt-1/ocm-arohcpci01-2tf7qg3qc9tr83rniscs197k8bo2fac2-o2o1a7a4r7x5y6r: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-94d2g5wkbdwn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3b3acad40780\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-94d2g5wkbdwn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9276160-mgmt-1/ocm-arohcpci01-2tf7qg3qc9tr83rniscs197k8bo2fac2-o2o1a7a4r7x5y6r: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-94d2g5wkbdwn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9276160-mgmt-1/ocm-arohcpci01-2tf7qg3qc9tr83rniscs197k8bo2fac2-o2o1a7a4r7x5y6r: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x3b3acbd1bd90\u003e{\n            s: \"dev/westus3/ci01-j9276160-mgmt-1/ocm-arohcpci01-2tf7qg3qc9tr83rniscs197k8bo2fac2-o2o1a7a4r7x5y6r: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7530/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110698197629276160","occurred_at":"2026-10-15T11:43:59Z","signature_id":"a427c0697c4081ce538b7aeb806c4fb09916f8e7070ed27c0d4d4512a3752ebe","pr_number":7530,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c7fba217b6ed5ce66bf9da85a1ad0fe4ef234d9b8e28c1d62c1a68359ce05dd8","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in \u003curl\u003e context deadline exceeded","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x128374ba040\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x128374ba020\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x128374ba040\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x128374ba020\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7498/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110305441572458496","occurred_at":"2026-10-14T09:43:19Z","signature_id":"7df74c111616fc166a1f2bc4df7a5d0c152bae783e433e710f514e16483b6958","pr_number":7498,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"dfd71f6e4c7fe588ab399e6a472c132eceea6fa6ee6f4e3aad318dc1c9c07df2","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected all","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w6xbc82hxr2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x280af577a000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w6xbc82hxr2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-2tf1898kdmei2jv6trajehsfpj8o4bk2-f7r5q8n1l4e4d9o: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w6xbc82hxr2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-2tf1898kdmei2jv6trajehsfpj8o4bk2-f7r5q8n1l4e4d9o: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0x280af63caa80\u003e{\n            s: \"dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-2tf1898kdmei2jv6trajehsfpj8o4bk2-f7r5q8n1l4e4d9o: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w6xbc82hxr2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x280af577a000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w6xbc82hxr2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-2tf1898kdmei2jv6trajehsfpj8o4bk2-f7r5q8n1l4e4d9o: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w6xbc82hxr2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-2tf1898kdmei2jv6trajehsfpj8o4bk2-f7r5q8n1l4e4d9o: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0x280af63caa80\u003e{\n            s: \"dev/centralus/ci01-j8670976-mgmt-2/ocm-arohcpci01-2tf1898kdmei2jv6trajehsfpj8o4bk2-f7r5q8n1l4e4d9o: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110588894188670976","occurred_at":"2026-10-15T04:29:39Z","signature_id":"5b412609e3463a02d7ca75b01131987fcae35acf585438d7204d1a174f6575bc","pr_number":6746,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a754bbeaf568a9f11e5562cb750f138c5a1e5121c5f07c9c3464251c13a17fc9","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in \u003curl\u003e context deadline exceeded","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32bc856472e0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x32bc85647260\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32bc856472e0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x32bc85647260\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110452257878183936","occurred_at":"2026-10-14T19:26:43Z","signature_id":"befc5c1e01c5061c2aad6d5d0c355f744c533fb274c0bab64d76e7eb496cca3b","pr_number":6746,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5207ee83cecbb4a62f10c9755387275f2a4075a267928e8754031a9cdbdeb408","failure_pattern":"after 4 attempts: 1 releases found in \u003curl\u003e which is not enough for the requested 1 offset","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:134]: failed to resolve configured channel candidate-5.1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xda3675c58e0\u003e: \n    after 4 attempts: 1 releases found in https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-5.1, which is not enough for the requested 1 offset\n    {\n        msg: \"after 4 attempts: 1 releases found in https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-5.1, which is not enough for the requested 1 offset\",\n        err: \u003c*errors.errorString | 0xda366e3dab0\u003e{\n            s: \"1 releases found in https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-5.1, which is not enough for the requested 1 offset\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:134]: failed to resolve configured channel candidate-5.1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xda3675c58e0\u003e: \n    after 4 attempts: 1 releases found in https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-5.1, which is not enough for the requested 1 offset\n    {\n        msg: \"after 4 attempts: 1 releases found in https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-5.1, which is not enough for the requested 1 offset\",\n        err: \u003c*errors.errorString | 0xda366e3dab0\u003e{\n            s: \"1 releases found in https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-5.1, which is not enough for the requested 1 offset\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","signature_id":"2ff884e11f43410ad920aa9b4baf8cb219805fce930bc1a46992b3fb5753fbd3","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":11,"prior_last_seen_at":"2026-10-07T22:28:41Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T20:53:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j5701632-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.167:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5701632/providers/microsoft.monitor/accounts/services-j5701632\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mxqgp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-15T21:19:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j5701632-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.167:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5701632/providers/microsoft.monitor/accounts/services-j5701632\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mxqgp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110810477125701632","occurred_at":"2026-10-15T19:10:09Z","signature_id":"d89d6f74cbb94ec931983a1c0f1751d925d097bdb28647a373945200520c857d","pr_number":7560,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1650432ef7f4a895179256b7f0a1a789aa6cd26091bc9f9d60b8d69099e131bf","failure_pattern":"alert [svc] KubeJobFailed fired","search_query":"alert [svc] KubeJobFailed fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeJobFailed does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeJobFailed does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T14:59:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeJobFailed\", cluster=\"ci01-j4159872-svc\", component=\"kubernetes-infrastructure\", condition=\"true\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.26:8080\", job=\"kube-state-metrics\", job_name=\"hubble-generate-certs-55c4c87c\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4159872/providers/microsoft.monitor/accounts/services-j4159872\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-mgfnh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Job kube-system/hubble-generate-certs-55c4c87c failed to complete. Removing failed job after investigation should clear this alert.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-14T14:59:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeJobFailed\", cluster=\"ci01-j4159872-svc\", component=\"kubernetes-infrastructure\", condition=\"true\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.26:8080\", job=\"kube-state-metrics\", job_name=\"hubble-generate-certs-55c4c87c\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4159872/providers/microsoft.monitor/accounts/services-j4159872\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-mgfnh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Job kube-system/hubble-generate-certs-55c4c87c failed to complete. Removing failed job after investigation should clear this alert."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7528/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110371307534159872","occurred_at":"2026-10-14T14:05:01Z","signature_id":"16c569152aa137a42851185754971c002c5a0136da3e260a036f60238b7422d2","pr_number":7528,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4a9080b1f90815862ee6922d98762609a1ba59b5059c78bbc7ca629997e6ab3f","failure_pattern":"alert [svc] PrometheusUptime fired","search_query":"alert [svc] PrometheusUptime fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert PrometheusUptime does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert PrometheusUptime does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-15T11:00:07Z\n  Ended:       2026-10-15T11:20:06Z\n  Severity:    Sev2\n  Labels:      alertname=\"PrometheusUptime\", cluster=\"ci01-j0312192-mgmt-2\", component=\"monitoring-infrastructure\", job=\"prometheus/prometheus\", namespace=\"prometheus\", region=\"centralus\", severity=\"critical\"\n  Description: Fewer than 95% of present Prometheus self-up samples were healthy over the past 24 hours.\nThis may indicate that the Prometheus server is down, experiencing network issues, or stuck in a crash loop.\nMissing samples are covered separately by PrometheusUptimeSampleCount.\nPlease check the status of the Prometheus pods, service endpoints, and network connectivity."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110665048610312192","occurred_at":"2026-10-15T09:32:14Z","signature_id":"e16deed747d65aec86614773f280752cb21849bd07bcf6f661e2a01fb205f914","pr_number":7467,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":89,"prior_last_seen_at":"2026-10-05T01:04:08Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","occurrences":1}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T16:40:12Z\n  Ended:       2026-10-12T16:50:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9626368-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-56bb7dd747-h9s6h\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container aro-hcp-backend in pod aro-hcp-backend-56bb7dd747-h9s6h (namespace aro-hcp) on cluster ci01-j9626368-svc has averaged 134.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-12T16:40:12Z\n  Ended:       2026-10-12T16:54:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9626368-mgmt-2\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-c6ff9b6f4-b8fh5\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container kube-applier in pod kube-applier-c6ff9b6f4-b8fh5 (namespace kube-applier) on cluster ci01-j9626368-mgmt-2 has averaged 150.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-12T16:40:12Z\n  Ended:       2026-10-12T16:56:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9626368-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-65fd757c56-d7zhs\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container kube-applier in pod kube-applier-65fd757c56-d7zhs (namespace kube-applier) on cluster ci01-j9626368-mgmt-1 has averaged 164.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"2ac18c0f28c3db988430e8cd508afbb6dc2ac259140c6a1cba13d912134a5113","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6685a5a6bc3f263f200f46fbec1e8851219602ada5b75c61eac4345b058adb70","failure_pattern":"alert [svc] userJourneyClusterDeletionErrors1h5m fired","search_query":"alert [svc] userJourneyClusterDeletionErrors1h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterDeletionErrors1h5m does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterDeletionErrors1h5m does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-16T13:12:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyClusterDeletionErrors1h5m\", cluster=\"ci01-j9203200-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9203200/providers/microsoft.monitor/accounts/services-j9203200\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"cluster-deletion-errors\"\n  Description: More than 72% of completed cluster delete operations on ci01-j9203200-svc failed or were canceled over the last hour with at least 3 failures, a 14.4x burn of the 95% SLO budget."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"bdc267b19eb19dceb0fc582a5893f5fe99d59172eb52ea317ffe1c61b8ab9c7e","pr_number":6982,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9e7d58cf80cefdbf58d438cd330cbadc1e097f5d57f87fa6133331546f9b5534","failure_pattern":"alert [svc] userJourneyClusterDeletionErrors6h30m fired","search_query":"alert [svc] userJourneyClusterDeletionErrors6h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterDeletionErrors6h30m does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterDeletionErrors6h30m does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-16T12:43:13Z\n  Ended:       2026-10-16T13:05:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyClusterDeletionErrors6h30m\", cluster=\"ci01-j9203200-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9203200/providers/microsoft.monitor/accounts/services-j9203200\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"cluster-deletion-errors\"\n  Description: More than 30% of completed cluster delete operations on ci01-j9203200-svc failed or were canceled over the last 6 hours (at least 5 completions), a 6x burn of the 95% SLO budget."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"e2f6ed75b3c3a7f5c74c0c161a3ab71c8a98b8f056b27a66eea5259563bedf3c","pr_number":6982,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b","failure_pattern":"alert [svc] userJourneyFrontendLatency6h30m fired","search_query":"alert [svc] userJourneyFrontendLatency6h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatency6h30m does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":7,"prior_last_seen_at":"2026-09-23T06:50:00Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatency6h30m does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-13T20:38:20Z\n  Ended:       2026-10-13T21:03:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatency6h30m\", cluster=\"ci01-j6273152-svc\", component=\"slo\", long_window=\"6h\", region=\"centralus\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n  Description: More than 6% of Frontend requests on cluster ci01-j6273152-svc exceeded 1s on both the 6h and 30m windows (6x burn of the 1% latency budget)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110083918836273152","occurred_at":"2026-10-13T19:02:36Z","signature_id":"751897ff3429e507ba41ef73f8187f9a0987de78f57b98795b44735681e9a017","pr_number":7505,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"fa97c78e78b0d10acd279c1a069b96e64dfa16a39f2711ce1dc8f939e8978246","failure_pattern":"cluster np-delete-hcp-cluster: expected 2 nodes, found 4; nodes per pool: np-deletable: [\u003cnode\u003e, \u003cnode\u003e], np-delete-main: [\u003cnode\u003e, \u003cnode\u003e]","search_query":"cluster np-delete-hcp-cluster: expected 2 nodes, found 4; nodes per pool: np-deletable: [u1i7j2r4v1j1a3a-np-deletable-hjjzc-b6p7m, u1i7j2r4v1j1a3a-np-deletable-hjjzc-rbs4q], np-delete-main: [u1i7j2r4v1j1a3a-np-delete-main-8wbbv-t4dx7, u1i7j2r4v1j1a3a-np-delete-main-8wbbv-wwsbq]","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:154]: failed to verify node count of 2 after deleting nodepool\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x922244cd220\u003e: \n    cluster np-delete-hcp-cluster: expected 2 nodes, found 4; nodes per pool: np-deletable: [u1i7j2r4v1j1a3a-np-deletable-hjjzc-b6p7m, u1i7j2r4v1j1a3a-np-deletable-hjjzc-rbs4q], np-delete-main: [u1i7j2r4v1j1a3a-np-delete-main-8wbbv-t4dx7, u1i7j2r4v1j1a3a-np-delete-main-8wbbv-wwsbq]\n    {\n        s: \"cluster np-delete-hcp-cluster: expected 2 nodes, found 4; nodes per pool: np-deletable: [u1i7j2r4v1j1a3a-np-deletable-hjjzc-b6p7m, u1i7j2r4v1j1a3a-np-deletable-hjjzc-rbs4q], np-delete-main: [u1i7j2r4v1j1a3a-np-delete-main-8wbbv-t4dx7, u1i7j2r4v1j1a3a-np-delete-main-8wbbv-wwsbq]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:154]: failed to verify node count of 2 after deleting nodepool\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x922244cd220\u003e: \n    cluster np-delete-hcp-cluster: expected 2 nodes, found 4; nodes per pool: np-deletable: [u1i7j2r4v1j1a3a-np-deletable-hjjzc-b6p7m, u1i7j2r4v1j1a3a-np-deletable-hjjzc-rbs4q], np-delete-main: [u1i7j2r4v1j1a3a-np-delete-main-8wbbv-t4dx7, u1i7j2r4v1j1a3a-np-delete-main-8wbbv-wwsbq]\n    {\n        s: \"cluster np-delete-hcp-cluster: expected 2 nodes, found 4; nodes per pool: np-deletable: [u1i7j2r4v1j1a3a-np-deletable-hjjzc-b6p7m, u1i7j2r4v1j1a3a-np-deletable-hjjzc-rbs4q], np-delete-main: [u1i7j2r4v1j1a3a-np-delete-main-8wbbv-t4dx7, u1i7j2r4v1j1a3a-np-delete-main-8wbbv-wwsbq]\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110772387883520000","occurred_at":"2026-10-15T16:38:43Z","signature_id":"bfc7e2cb88e3c3c3860a3405202897e02247a9d1db4c4b3a10d3d62d11bc6ad3","pr_number":7560,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"599d126c4f18389701726c36f576de53996cc8bf35c7455f30ca19957cc25b45","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"3vAWYLSpgMpdmMCKnkDG1+GD7QF9y2L9E9AOfsGsY04=\"","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"3vAWYLSpgMpdmMCKnkDG1+GD7QF9y2L9E9AOfsGsY04=\"","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:364]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x181b25f73380\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"3vAWYLSpgMpdmMCKnkDG1+GD7QF9y2L9E9AOfsGsY04=\"\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \\\"3vAWYLSpgMpdmMCKnkDG1+GD7QF9y2L9E9AOfsGsY04=\\\"\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:364]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x181b25f73380\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"3vAWYLSpgMpdmMCKnkDG1+GD7QF9y2L9E9AOfsGsY04=\"\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \\\"3vAWYLSpgMpdmMCKnkDG1+GD7QF9y2L9E9AOfsGsY04=\\\"\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7500/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110389003348873216","occurred_at":"2026-10-14T15:15:22Z","signature_id":"586406f9efd0bd6dcafe1c3adb0dee6382f6f3192b935a25b350e0fba5d752ea","pr_number":7500,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aad982042b35cafe44f073a348d1c5f0c470b4cc00a2366a7f00c95c50f36d2e","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j0731648-mgmt-1\",\"clusterName\":\"ci01-j073164","search_query":"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-15T10:41:48.83104699Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0731648-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0731648-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-15T10:41:50.721098398Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0731648-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0731648-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-15T10:41:51.099462414Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0731648-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0731648-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-10-15T10:41:48.821Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-10-15T11:11:48.321Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-15T10:41:48.83104699Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0731648-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0731648-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-15T10:41:50.721098398Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0731648-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0731648-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-15T10:41:51.099462414Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0731648-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0731648-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7551/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110673040600731648","occurred_at":"2026-10-15T10:04:01Z","signature_id":"55419a4cb8cad837215b41faf77ddc09a1f7332df70bda276b4516d308d72fc7","pr_number":7551,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4d876456a841781d4034f0b3b80449cf5b52a93b8aafda5fb51135d2c9b1e459","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j2981888-mgmt-1\",\"clusterName\":\"ci01-j298188","search_query":"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-14T19:33:50.055321564Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2981888-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2981888-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-14T19:33:52.298968498Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2981888-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2981888-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-14T19:33:52.677544841Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2981888-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2981888-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-10-14T19:33:50.044Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-10-14T20:03:49.640Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-14T19:33:50.055321564Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2981888-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2981888-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-14T19:33:52.298968498Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2981888-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2981888-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-14T19:33:52.677544841Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2981888-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2981888-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6946/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110441834562981888","occurred_at":"2026-10-14T18:45:15Z","signature_id":"3413e51414076b21936c915303e7fa16ef8b8cf4ecc8fff350a16108a153e807","pr_number":6946,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"75ba2f620eb9a6826cee085233900a138186175271fd9f54c13e5c69036cc8d1","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j8714496-mgmt-1\",\"clusterName\":\"ci01-j871449","search_query":"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-16T14:04:27.363830736Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8714496-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j8714496-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-16T14:04:29.161200885Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8714496-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j8714496-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-16T14:04:29.579255531Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8714496-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j8714496-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-10-16T14:04:27.353Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-10-16T14:34:26.879Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-16T14:04:27.363830736Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8714496-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j8714496-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-16T14:04:29.161200885Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8714496-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j8714496-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-16T14:04:29.579255531Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8714496-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j8714496-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111087082628714496","occurred_at":"2026-10-16T13:29:17Z","signature_id":"2a6bf4d1ff2fafa7ef97169789e44726bf300fcadbf9dec849e1efa942bd4a0d","pr_number":7232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cca9ba181eaef5e426cb8fe33160e83ad67f9d053e87b16b9d8d8cbf3c5cc14d","failure_pattern":"failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs","search_query":"failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 602.098s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3618a9570d00\u003e: \n    failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\n    | where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\n    | distinct cid, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": net/http: TLS handshake timeout\n    {\n        msg: \"failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n        err: \u003c*fmt.wrapError | 0x3618a9570cc0\u003e{\n            msg: \"failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n            err: \u003c*fmt.wrapError | 0x3618a9570c60\u003e{\n                msg: \"failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n                err: \u003c*fmt.wrapError | 0x3618a9570c40\u003e{\n                    msg: \"failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n                    err: \u003c*errors.Error | 0x3618a9654280\u003e{\n                        Op: 1,\n                        Kind: 7,\n                        Err: \u003c*fmt.wrapError | 0x3618a9570c00\u003e{\n                            msg: \"With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n                            err: \u003c*url.Error | 0x3618aa1308a0\u003e{\n                                Op: \"Post\",\n                                URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                                Err: \u003chttp.tlsHandshakeTimeoutError\u003e{},\n                            },\n                        },\n                        restErrMsg: nil,\n                        decoded: nil,\n                        permanent: false,\n                        inner: nil,\n                    },\n                },\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 602.098s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3618a9570d00\u003e: \n    failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\n    | where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\n    | distinct cid, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": net/http: TLS handshake timeout\n    {\n        msg: \"failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n        err: \u003c*fmt.wrapError | 0x3618a9570cc0\u003e{\n            msg: \"failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n            err: \u003c*fmt.wrapError | 0x3618a9570c60\u003e{\n                msg: \"failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n                err: \u003c*fmt.wrapError | 0x3618a9570c40\u003e{\n                    msg: \"failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n                    err: \u003c*errors.Error | 0x3618a9654280\u003e{\n                        Op: 1,\n                        Kind: 7,\n                        Err: \u003c*fmt.wrapError | 0x3618a9570c00\u003e{\n                            msg: \"With query: clustersServiceLogs\\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\\n| distinct cid, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": net/http: TLS handshake timeout\",\n                            err: \u003c*url.Error | 0x3618aa1308a0\u003e{\n                                Op: \"Post\",\n                                URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                                Err: \u003chttp.tlsHandshakeTimeoutError\u003e{},\n                            },\n                        },\n                        restErrMsg: nil,\n                        decoded: nil,\n                        permanent: false,\n                        inner: nil,\n                    },\n                },\n            },\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110261282979450880","occurred_at":"2026-10-14T06:47:51Z","signature_id":"d94811361b1b2c494889c4f4d830e8abab41778eec15e79fdb85a81df8ec3ea3","pr_number":6712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1b9753d6df9613da46b4b9cfaf32730b876169734288bd96a4ec92c7f1a772fa","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.002s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2582980c0060\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\n    let _endTime = datetime(2026-10-12T11:53:04.2658821Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-82f6cfwmh699'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-12T11:53:10.3021968Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\", ActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", SubActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", ActivityStack:\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x258297402990\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x2582980c0040\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x2582980c0020\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x2582961135c0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x258296113580\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x258296cdc200\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.002s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2582980c0060\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\n    let _endTime = datetime(2026-10-12T11:53:04.2658821Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-82f6cfwmh699'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-12T11:53:10.3021968Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\", ActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", SubActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", ActivityStack:\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x258297402990\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x2582980c0040\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x2582980c0020\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x2582961135c0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x258296113580\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x258296cdc200\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7420/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109588388414754816","occurred_at":"2026-10-12T10:14:00Z","signature_id":"670cbec864bb9c486cb9441591ed8f63fe3036fd2cb244750889987cfff959c9","pr_number":7420,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"deea4d7ee03e8432bcb988590b2236f74b08241de4cdde1789d1247ae2f6a03f","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.008s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x39ffd3acaac0\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\n    let _endTime = datetime(2026-10-13T21:06:23.2601111Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-13T21:06:33.6034532Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\", ActivityId:\"91f407a2-0a4c-43e0-acd3-c6532e675296\", SubActivityId:\"91f407a2-0a4c-43e0-acd3-c6532e675296\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"91f407a2-0a4c-43e0-acd3-c6532e675296\", ActivityStack:\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x39ffd2546540\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x39ffd3acaaa0\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x39ffd3acaa80\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x39ffd0047e00\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x39ffd0047dc0\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x39ffd0960100\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.008s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x39ffd3acaac0\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\n    let _endTime = datetime(2026-10-13T21:06:23.2601111Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-13T21:06:33.6034532Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\", ActivityId:\"91f407a2-0a4c-43e0-acd3-c6532e675296\", SubActivityId:\"91f407a2-0a4c-43e0-acd3-c6532e675296\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"91f407a2-0a4c-43e0-acd3-c6532e675296\", ActivityStack:\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x39ffd2546540\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x39ffd3acaaa0\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x39ffd3acaa80\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x39ffd0047e00\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x39ffd0047dc0\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x39ffd0960100\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110091526691360768","occurred_at":"2026-10-13T19:28:19Z","signature_id":"f25671b9f4bd359b018cb9e9ac954a66d4ed6290a26895e1fd925ac9f22976fe","pr_number":7492,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"8baf31258527a9c37481b02b6c4e240b9ba893342f978c453787fc7905c9d95b","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 604.992s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x39f5f91e0c80\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\n    let _endTime = datetime(2026-10-13T23:09:06.1094687Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-13T23:09:29.6842855Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\", ActivityId:\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\", SubActivityId:\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x39f5f8768288\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x39f5f91e0c60\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x39f5f91e0c40\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x39f5f91e0ba0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x39f5f91e0b60\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x39f5f6b7ee00\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 604.992s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x39f5f91e0c80\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\n    let _endTime = datetime(2026-10-13T23:09:06.1094687Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-13T23:09:29.6842855Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\", ActivityId:\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\", SubActivityId:\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x39f5f8768288\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x39f5f91e0c60\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x39f5f91e0c40\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x39f5f91e0ba0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x39f5f91e0b60\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x39f5f6b7ee00\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110123903912251392","occurred_at":"2026-10-13T21:41:57Z","signature_id":"8b0aac51428f6336203e5c99b582b7417a3d1fd3f1323459c75ebe8411a4f273","pr_number":7488,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"38232e514c86bfe6cab965d283fee05c3ce21024c2a543efb679b24a42cf1009","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1e936bab0320\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\n    let _endTime = datetime(2026-10-14T02:13:05.3130233Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-zxm6njn97hd4'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T02:13:17.5036831Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\", ActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", SubActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x1e936b180db0\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x1e936bab0300\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x1e936bab02e0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x1e93699ddd20\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x1e93699ddc60\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x1e936a2c1400\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1e936bab0320\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\n    let _endTime = datetime(2026-10-14T02:13:05.3130233Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-zxm6njn97hd4'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T02:13:17.5036831Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\", ActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", SubActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x1e936b180db0\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x1e936bab0300\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x1e936bab02e0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x1e93699ddd20\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x1e93699ddc60\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x1e936a2c1400\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110168614748295168","occurred_at":"2026-10-14T00:39:37Z","signature_id":"83eb0b332787f68fc85dd200d810732ea20732bc14eafa3beefcaaf8deb483c0","pr_number":7509,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b6abc857fe5890451778a1e8bd584c9cdff0daecc370ddea658871d7ad261b7b","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1fce6bc96f40\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\n    let _endTime = datetime(2026-10-14T03:48:07.3491058Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-sbj59nsxlhq5'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T03:48:19.7777147Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\", ActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", SubActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x1fce6bb2e558\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x1fce6bc96f20\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x1fce6bc96f00\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x1fce6b3e4d40\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x1fce6b3e4d00\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x1fce6d014600\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1fce6bc96f40\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\n    let _endTime = datetime(2026-10-14T03:48:07.3491058Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-sbj59nsxlhq5'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T03:48:19.7777147Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\", ActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", SubActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x1fce6bb2e558\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x1fce6bc96f20\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x1fce6bc96f00\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x1fce6b3e4d40\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x1fce6b3e4d00\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x1fce6d014600\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7510/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110189966687473664","occurred_at":"2026-10-14T02:04:28Z","signature_id":"a63d52fcb7a18aef09e6286c67a8d5f921df82fc1ef1c88cc7261cf7af793684","pr_number":7510,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7db277832781bbf28aea2476274ab473a26eb0485a79733307a4e7919213f9d8","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x356d91091ca0\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\n    let _endTime = datetime(2026-10-14T19:48:15.7310678Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T19:48:28.4804395Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\", ActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", SubActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x356d921ef170\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x356d91091c80\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x356d91091c60\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x356d91091b60\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x356d91091b20\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x356d9135a600\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x356d91091ca0\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\n    let _endTime = datetime(2026-10-14T19:48:15.7310678Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T19:48:28.4804395Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\", ActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", SubActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x356d921ef170\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x356d91091c80\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x356d91091c60\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x356d91091b60\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x356d91091b20\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T19:48:28.4804395Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\\\", ActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", SubActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b \u003e GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x356d9135a600\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110436010226618368","occurred_at":"2026-10-14T18:22:06Z","signature_id":"79ceb6ec46d015e2d8448735d45968be712ed99bbb1e9bded870eb184f6c9787","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"09c6c22177bc75493a0fe4d10381237aa66c36f43bae5c0efd353ae0bc7fe378","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 611.066s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x32cad0a314a0\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\n    let _endTime = datetime(2026-10-14T23:11:15.1642696Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-lznjhs5kd54t'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T23:11:29.6178254Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\", ActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", SubActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x32cad17d37b8\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x32cad0a31480\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x32cad0a31460\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x32cad0a313a0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x32cad0a31360\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x32cacfbb4000\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 611.066s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x32cad0a314a0\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\n    let _endTime = datetime(2026-10-14T23:11:15.1642696Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-lznjhs5kd54t'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T23:11:29.6178254Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\", ActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", SubActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x32cad17d37b8\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x32cad0a31480\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x32cad0a31460\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x32cad0a313a0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x32cad0a31360\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T23:11:29.6178254Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\\\", ActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", SubActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 \u003e GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x32cacfbb4000\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7524/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110483335657558016","occurred_at":"2026-10-14T21:30:12Z","signature_id":"9f1df4172af337452f2016a62aa2acc119ed3772d4f79dec7f55d22bea8822df","pr_number":7524,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b1d26f5c0f0ecd0703defbf54e63749a77af0d4139d2616692f6f3a5d3b3dda0","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 607.834s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1b1ea5606860\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\n    let _endTime = datetime(2026-10-15T13:36:27.6218063Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-nzl7t56rlmwg'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T13:36:43.8312817Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\", ActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", SubActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityStack:\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb \u003e GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T13:36:43.8312817Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\\\", ActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", SubActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb \u003e GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x1b1ea82ed560\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x1b1ea56067e0\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T13:36:43.8312817Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\\\", ActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", SubActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb \u003e GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x1b1ea56067c0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T13:36:43.8312817Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\\\", ActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", SubActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb \u003e GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x1b1ea4c841e0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T13:36:43.8312817Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\\\", ActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", SubActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb \u003e GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x1b1ea4c84000\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T13:36:43.8312817Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\\\", ActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", SubActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb \u003e GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x1b1ea51ea200\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 607.834s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1b1ea5606860\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\n    let _endTime = datetime(2026-10-15T13:36:27.6218063Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-nzl7t56rlmwg'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T13:36:43.8312817Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\", ActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", SubActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\", ActivityStack:\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb \u003e GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T13:36:43.8312817Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\\\", ActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", SubActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb \u003e GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x1b1ea82ed560\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x1b1ea56067e0\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T13:36:43.8312817Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\\\", ActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", SubActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb \u003e GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x1b1ea56067c0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T13:36:43.8312817Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\\\", ActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", SubActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb \u003e GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x1b1ea4c841e0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T13:36:43.8312817Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\\\", ActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", SubActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb \u003e GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x1b1ea4c84000\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T13:36:27.6218062Z);\\nlet _endTime = datetime(2026-10-15T13:36:27.6218063Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nzl7t56rlmwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T13:36:43.8312817Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07\\\", ActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", SubActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"20e50d07-291a-4b8a-941b-eeee87c2d5bb\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;11535679-bceb-4892-8ed7-c3b0c6e50f07 ARID=20e50d07-291a-4b8a-941b-eeee87c2d5bb \u003e GW.Http.CallContext/20e50d07-291a-4b8a-941b-eeee87c2d5bb)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x1b1ea51ea200\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7527/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110702760532905984","occurred_at":"2026-10-15T12:01:55Z","signature_id":"72220dc9a27ede8d1cc61beb86aa461b268e8b724e5666fc7830d392d8da3185","pr_number":7527,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"6a2faa8ff03113556c7e7642d14f84e8ee85374b46ab8246803fea2cb68fb836","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x11bd47e7c4a0\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\n    let _endTime = datetime(2026-10-15T14:26:17.9586903Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T14:26:30.5366490Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\", ActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", SubActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e \u003e GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T14:26:30.5366490Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\\\", ActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", SubActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e \u003e GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x11bd532fd080\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x11bd47e7c480\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T14:26:30.5366490Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\\\", ActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", SubActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e \u003e GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x11bd47e7c440\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T14:26:30.5366490Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\\\", ActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", SubActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e \u003e GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x11bd49601a80\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T14:26:30.5366490Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\\\", ActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", SubActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e \u003e GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x11bd49601a20\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T14:26:30.5366490Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\\\", ActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", SubActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e \u003e GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x11bd4901a900\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x11bd47e7c4a0\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\n    let _endTime = datetime(2026-10-15T14:26:17.9586903Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T14:26:30.5366490Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\", ActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", SubActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e \u003e GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T14:26:30.5366490Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\\\", ActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", SubActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e \u003e GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x11bd532fd080\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x11bd47e7c480\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T14:26:30.5366490Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\\\", ActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", SubActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e \u003e GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x11bd47e7c440\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T14:26:30.5366490Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\\\", ActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", SubActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e \u003e GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x11bd49601a80\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T14:26:30.5366490Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\\\", ActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", SubActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e \u003e GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x11bd49601a20\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T14:26:17.9586902Z);\\nlet _endTime = datetime(2026-10-15T14:26:17.9586903Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-hzvnhdkdhrhv'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T14:26:30.5366490Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10\\\", ActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", SubActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"53ab61d9-b679-46ee-bc5b-6292ddb0a06e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;a1f01281-ebd6-4591-8838-73e0f2499c10 ARID=53ab61d9-b679-46ee-bc5b-6292ddb0a06e \u003e GW.Http.CallContext/53ab61d9-b679-46ee-bc5b-6292ddb0a06e)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x11bd4901a900\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110717161084293120","occurred_at":"2026-10-15T12:59:19Z","signature_id":"71d9d576a1a8fca56957c8b5cd037779b315db23a013dab6b5e7c85b301fd90a","pr_number":7557,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4254d25e1bb34fadc6f2ec3b79f5caf40486f57682b63c4fa81efaf0f7965761","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 610.563s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x37a644e389e0\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\n    let _endTime = datetime(2026-10-15T15:31:15.7400199Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-qsncztkggf76'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T15:31:33.5544050Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\", ActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", SubActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 \u003e GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qsncztkggf76'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T15:31:33.5544050Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\\\", ActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", SubActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 \u003e GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x37a64583fda0\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x37a644e389c0\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qsncztkggf76'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T15:31:33.5544050Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\\\", ActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", SubActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 \u003e GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x37a644e389a0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qsncztkggf76'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T15:31:33.5544050Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\\\", ActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", SubActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 \u003e GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x37a645c4b2c0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qsncztkggf76'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T15:31:33.5544050Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\\\", ActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", SubActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 \u003e GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x37a645c4b260\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qsncztkggf76'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T15:31:33.5544050Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\\\", ActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", SubActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 \u003e GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x37a646042000\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 610.563s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x37a644e389e0\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\n    let _endTime = datetime(2026-10-15T15:31:15.7400199Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-qsncztkggf76'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T15:31:33.5544050Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\", ActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", SubActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 \u003e GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qsncztkggf76'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T15:31:33.5544050Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\\\", ActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", SubActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 \u003e GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x37a64583fda0\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x37a644e389c0\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qsncztkggf76'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T15:31:33.5544050Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\\\", ActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", SubActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 \u003e GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x37a644e389a0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qsncztkggf76'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T15:31:33.5544050Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\\\", ActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", SubActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 \u003e GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x37a645c4b2c0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qsncztkggf76'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T15:31:33.5544050Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\\\", ActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", SubActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 \u003e GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x37a645c4b260\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T15:31:15.7400196Z);\\nlet _endTime = datetime(2026-10-15T15:31:15.7400199Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-qsncztkggf76'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T15:31:33.5544050Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49\\\", ActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", SubActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;3a6a80f8-130a-4c8f-96c2-ad27a8b6cf49 ARID=9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8 \u003e GW.Http.CallContext/9f0ff6bc-758a-4b0e-97c1-d6daa0a36ad8)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x37a646042000\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7489/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110733173536067584","occurred_at":"2026-10-15T14:02:58Z","signature_id":"c1b0bf2487d27b5de2f3687c30bcbf70c9723dec17cef33db049e1eba4c4b01f","pr_number":7489,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"df767f417b73cfd615fb200938491ed2d5e2c58ba2ba60fd6a8c9619fdc3b6c1","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 615.207s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x55a339ed620\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\n    let _endTime = datetime(2026-10-15T17:04:06.8555808Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:04:20.6431913Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\", ActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", SubActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityStack:\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 \u003e GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:04:20.6431913Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\\\", ActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", SubActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 \u003e GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x55a33cca5a0\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x55a339ed600\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:04:20.6431913Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\\\", ActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", SubActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 \u003e GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x55a339ed5e0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:04:20.6431913Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\\\", ActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", SubActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 \u003e GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x55a331fa980\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:04:20.6431913Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\\\", ActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", SubActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 \u003e GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x55a331fa920\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:04:20.6431913Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\\\", ActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", SubActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 \u003e GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x55a32a9e000\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 615.207s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x55a339ed620\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\n    let _endTime = datetime(2026-10-15T17:04:06.8555808Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T17:04:20.6431913Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\", ActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", SubActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\", ActivityStack:\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 \u003e GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:04:20.6431913Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\\\", ActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", SubActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 \u003e GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x55a33cca5a0\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x55a339ed600\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:04:20.6431913Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\\\", ActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", SubActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 \u003e GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x55a339ed5e0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:04:20.6431913Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\\\", ActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", SubActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 \u003e GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x55a331fa980\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:04:20.6431913Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\\\", ActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", SubActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 \u003e GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x55a331fa920\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T17:04:06.8555806Z);\\nlet _endTime = datetime(2026-10-15T17:04:06.8555808Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-rdrv6kmh5rwg'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T17:04:20.6431913Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3\\\", ActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", SubActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;57c1ddce-6f6c-4d21-880d-fa0824bc6dc3 ARID=1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20 \u003e GW.Http.CallContext/1a2974b8-7e9f-40fc-a27f-9efbb4cd4b20)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x55a32a9e000\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7561/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110756784728707072","occurred_at":"2026-10-15T15:36:36Z","signature_id":"df65f9d3f723fb1f700d3e4512bc24cfe15d06d3f717d720adb86c6def5fd493","pr_number":7561,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6c583e3c50547204ea3ec4d9e3f5bb252785d0c2a8558c68d18e7ab44131cc61","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x698a7204a40\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\n    let _endTime = datetime(2026-10-15T20:56:57.7600018Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-strwsplkk89b'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T20:57:10.6107763Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\", ActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", SubActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a \u003e GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-strwsplkk89b'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T20:57:10.6107763Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\\\", ActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", SubActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a \u003e GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x698a61c09c0\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x698a7204a20\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-strwsplkk89b'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T20:57:10.6107763Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\\\", ActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", SubActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a \u003e GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x698a7204a00\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-strwsplkk89b'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T20:57:10.6107763Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\\\", ActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", SubActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a \u003e GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x698a6aac4a0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-strwsplkk89b'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T20:57:10.6107763Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\\\", ActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", SubActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a \u003e GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x698a6aac460\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-strwsplkk89b'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T20:57:10.6107763Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\\\", ActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", SubActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a \u003e GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x698a632ba00\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x698a7204a40\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\n    let _endTime = datetime(2026-10-15T20:56:57.7600018Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-strwsplkk89b'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T20:57:10.6107763Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\", ActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", SubActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\", ActivityStack:\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a \u003e GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-strwsplkk89b'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T20:57:10.6107763Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\\\", ActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", SubActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a \u003e GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x698a61c09c0\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x698a7204a20\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-strwsplkk89b'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T20:57:10.6107763Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\\\", ActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", SubActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a \u003e GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x698a7204a00\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-strwsplkk89b'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T20:57:10.6107763Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\\\", ActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", SubActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a \u003e GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x698a6aac4a0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-strwsplkk89b'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T20:57:10.6107763Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\\\", ActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", SubActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a \u003e GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x698a6aac460\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T20:56:57.7600016Z);\\nlet _endTime = datetime(2026-10-15T20:56:57.7600018Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-strwsplkk89b'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T20:57:10.6107763Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d\\\", ActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", SubActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0b2f5fe2-6a96-4579-a835-e7b034bb165a\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;afc59ee6-47b1-435b-889a-55648a3da78d ARID=0b2f5fe2-6a96-4579-a835-e7b034bb165a \u003e GW.Http.CallContext/0b2f5fe2-6a96-4579-a835-e7b034bb165a)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x698a632ba00\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110810464379211776","occurred_at":"2026-10-15T19:10:01Z","signature_id":"115561144c664a4ac802c2fb2259cec80596351f3f66e6fb77db0868559bbba9","pr_number":7373,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1a07c89b1a2911e28f6554f99ebb94d6ecf5dae243f8e12903ed1a621a5c96ab","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.006s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x39916b3f7320\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\n    let _endTime = datetime(2026-10-15T21:07:31.8528726Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-54zkzjslmt86'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:07:45.2821972Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\", ActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", SubActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityStack:\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 \u003e GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:07:45.2821972Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\\\", ActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", SubActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 \u003e GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x39916c0fc5a0\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x39916b3f7300\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:07:45.2821972Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\\\", ActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", SubActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 \u003e GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x39916b3f72e0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:07:45.2821972Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\\\", ActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", SubActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 \u003e GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x39916b897900\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:07:45.2821972Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\\\", ActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", SubActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 \u003e GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x39916b8978c0\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:07:45.2821972Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\\\", ActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", SubActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 \u003e GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x3991698bcf00\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.006s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x39916b3f7320\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\n    let _endTime = datetime(2026-10-15T21:07:31.8528726Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-54zkzjslmt86'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T21:07:45.2821972Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\", ActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", SubActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"203611ca-e380-4c98-a4a7-665b077207f6\", ActivityStack:\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 \u003e GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:07:45.2821972Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\\\", ActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", SubActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 \u003e GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x39916c0fc5a0\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x39916b3f7300\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:07:45.2821972Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\\\", ActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", SubActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 \u003e GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x39916b3f72e0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:07:45.2821972Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\\\", ActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", SubActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 \u003e GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x39916b897900\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:07:45.2821972Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\\\", ActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", SubActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 \u003e GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x39916b8978c0\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-14T21:07:31.8528725Z);\\nlet _endTime = datetime(2026-10-15T21:07:31.8528726Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-54zkzjslmt86'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-15T21:07:45.2821972Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b\\\", ActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", SubActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"203611ca-e380-4c98-a4a7-665b077207f6\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;4ae8ea33-2e52-4cf0-9231-dbde5018ba4b ARID=203611ca-e380-4c98-a4a7-665b077207f6 \u003e GW.Http.CallContext/203611ca-e380-4c98-a4a7-665b077207f6)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x3991698bcf00\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110813601194315776","occurred_at":"2026-10-15T19:22:32Z","signature_id":"26411655518cdd0c20fcf81ac23aaa2f9e93b979531413e5572c2458b3d349d3","pr_number":6928,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"455d8d83ccb887ae2fa2ed5ea0584fa15fadc3fdfb663655f7a7e68cc891140d","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xd373b6d7240\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\n    let _endTime = datetime(2026-10-16T00:41:03.5278907Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-wscgxrnrv56t'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-16T00:41:17.7915220Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\", ActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", SubActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b \u003e GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-16T00:41:17.7915220Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\\\", ActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", SubActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b \u003e GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0xd373b6616c8\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0xd373b6d7220\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-16T00:41:17.7915220Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\\\", ActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", SubActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b \u003e GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0xd373b6d7200\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-16T00:41:17.7915220Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\\\", ActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", SubActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b \u003e GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0xd373b6d7140\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-16T00:41:17.7915220Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\\\", ActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", SubActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b \u003e GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0xd373b6d7100\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-16T00:41:17.7915220Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\\\", ActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", SubActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b \u003e GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0xd3739a6a100\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xd373b6d7240\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\n    let _endTime = datetime(2026-10-16T00:41:03.5278907Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-wscgxrnrv56t'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-16T00:41:17.7915220Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\", ActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", SubActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"94321735-0ea3-4275-97c8-e484b256af4b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b \u003e GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-16T00:41:17.7915220Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\\\", ActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", SubActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b \u003e GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0xd373b6616c8\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0xd373b6d7220\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-16T00:41:17.7915220Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\\\", ActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", SubActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b \u003e GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0xd373b6d7200\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-16T00:41:17.7915220Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\\\", ActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", SubActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b \u003e GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0xd373b6d7140\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-16T00:41:17.7915220Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\\\", ActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", SubActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b \u003e GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0xd373b6d7100\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-15T00:41:03.5278905Z);\\nlet _endTime = datetime(2026-10-16T00:41:03.5278907Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-wscgxrnrv56t'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-16T00:41:17.7915220Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310\\\", ActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", SubActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"94321735-0ea3-4275-97c8-e484b256af4b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;46087c04-d93a-4e75-8b87-4f6098fc8310 ARID=94321735-0ea3-4275-97c8-e484b256af4b \u003e GW.Http.CallContext/94321735-0ea3-4275-97c8-e484b256af4b)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0xd3739a6a100\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110871399009619968","occurred_at":"2026-10-15T23:12:14Z","signature_id":"7384fa885a9f87432a6faf48777d0d2a5f7012e19263c7917f690160a72f39aa","pr_number":6746,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"defe5d468c26c44c499810219b81a1dcd05b142f28a898794885f56e139df072","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-07T13:55:41Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-14T16:56:13.797Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 description=\"Step deploy-mce-config\\n Kind: Helm\\n\"\ntime=2026-10-14T16:56:14.254Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157 global:map[imageOverrides:map[governance_policy_propagator:governance-policy-propagator-acm-216@sha256:deb737b19932d520553c902ee5e0738ebeaad4592e8aa9649c41fec6fc00b62e] registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-10-14T16:56:14.254Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2\ntime=2026-10-14T16:56:14.292Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 namespace=multicluster-engine\ntime=2026-10-14T16:56:14.362Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 dryRun=false\ntime=2026-10-14T16:56:14.362Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2\ntime=2026-10-14T16:56:14.432Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2\ntime=2026-10-14T17:06:42.230Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 err=\"failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\"\ntime=2026-10-14T17:06:42.231Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110398398338174976","occurred_at":"2026-10-14T15:52:39Z","signature_id":"4fbcbcae7d0f352f867128831decf8a2cd0c0541228c3392fa864aa36c4d6550","pr_number":7509,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f8c211741afc2e8a543c9d12a5fbe95ef4d060ea80fdf9d473e8c25c212fda2f","failure_pattern":"failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 14/15 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 14/15\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kubelet-ds (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kubelet-ds (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-10-15T17:33:07.002Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1 description=\"Step kubelet-ds\\n Kind: Helm\\n\"\ntime=2026-10-15T17:33:07.498Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1 values=map[enabled:true]\ntime=2026-10-15T17:33:07.498Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1\ntime=2026-10-15T17:33:07.598Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1 namespace=kubelet-ds\ntime=2026-10-15T17:33:07.719Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1 dryRun=false\ntime=2026-10-15T17:33:07.719Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1\ntime=2026-10-15T17:33:07.774Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1\ntime=2026-10-15T17:38:08.700Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1 err=\"resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 14/15\\ncontext deadline exceeded\"\ntime=2026-10-15T17:38:08.700Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 14/15\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110779912934658048","occurred_at":"2026-10-15T17:08:42Z","signature_id":"0842be46fe2b3ebe671974ce35c826abb71f7583993632c9a306d69aa8ced78e","pr_number":7509,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b3e3dde3ed6e86eca14ea52f55b0d41a64a9d455bb9f2e88e86dfa581de65743","failure_pattern":"failed to roll out Helm release: resource Deployment/acrpull/acrpull not ready. status: InProgress, message: Available: 1/2 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/acrpull/acrpull not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/acrpull","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/acrpull","occurrences":1}],"full_error_samples":["time=2026-10-12T10:17:07.734Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull description=\"Step acrpull\\n Kind: Helm\\n\"\ntime=2026-10-12T10:17:08.348Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull values=\"map[image:mcr.microsoft.com/aks/msi-acrpull@sha256:b0cbba7044d5142f8545758e5a69b7b35e7f8c6af596bc972e164d2b345978a7 namespace:acrpull replicas:2 ttlRotationFraction:0.5]\"\ntime=2026-10-12T10:17:08.348Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull\ntime=2026-10-12T10:17:08.589Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull namespace=acrpull\ntime=2026-10-12T10:17:08.838Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull dryRun=false\ntime=2026-10-12T10:17:08.838Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull\ntime=2026-10-12T10:17:08.976Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull\ntime=2026-10-12T10:22:16.134Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull err=\"resource Deployment/acrpull/acrpull not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\"\ntime=2026-10-12T10:22:16.135Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/acrpull/acrpull not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109576437055885312","occurred_at":"2026-10-12T09:26:31Z","signature_id":"5ac3f50227d6a7076813d8bbbe9de0e8c8876a26e6800117bb2654c1677cfb10","pr_number":7466,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a8bb4542e08df91dbf01364e592eda2b949d63ea36dbf43a83cbdbd4c51b7dfe","failure_pattern":"failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-03T00:45:21Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","occurrences":1}],"full_error_samples":["time=2026-10-16T16:06:45.672Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-16T16:06:46.344Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy values=\"map[azureArmHelperIdentityCertName:armHelperPoolCert-38 azureArmHelperIdentityClientId:6608bb27-dfa1-4c02-8395-4ca010bd73db azureArmHelperMockFpaPrincipalId:47f69502-0065-4d9a-b19b-d403e183d2f4 azureCloudEnvironmentName:AzurePublicCloud azureCsMiClientId:d0cc3060-1e84-4147-a32a-544fe7e53387 azureFirstPartyApplicationClientId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX azureMiMockServicePrincipalCertName:msiMockPoolCert-16 azureMiMockServicePrincipalClientId:33e65307-a696-491e-831f-42341ce3a9b5 azureMiMockServicePrincipalPrincipalId:b26c2343-0863-4a55-bb7e-dfb2544999c1 azureOperatorsMI:map[dev:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] public:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] roleSetName:dev] backplaneURL: batchProcesses: batchProcessesDryRun:true clientId:foo clientScopes:openid clientSecret:bar clusterErrorReport:false cosmosDatabaseName:arohcpci01-rp-j7660416 cosmosDatabaseUrl:https://arohcpci01-rp-j7660416.documents.azure.com:443/ cpuRequest:630m csDeploymentStrategy:map[rollingUpdate:map[maxSurge:1 maxUnavailable:0] type:RollingUpdate] dataPlaneHAProxyImage:arohcpocpdev.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678 databaseAuthMethod:postgres databaseDisableTls:true databaseHost:ocm-cs-db databaseK8sSecretName:ocm-cs-db databaseName:ocm-cs-db databasePassword:TheBlurstOfTimes databasePort:5432 databaseServiceName:ocm-cs-db databaseUser:ocm databaseVolumeCapacity:512Mi debugMaxDuration:5m debugPort: defaultExpiration:0 deployment:map[zoneCount:3] environment:arohcpci01 forceMigration: fpaCertName:firstPartyCert2 gatewayUrl:http://127.0.0.1:9090 global:map[imageDigest:sha256:e9d0c188179de6a6eb745bf8b91e2a9cf54424a74b55ce29ef87ab06cf658ddb] imageDigest:sha256:e9d0c188179de6a6eb745bf8b91e2a9cf54424a74b55ce29ef87ab06cf658ddb imageRegistry:arohcpsvcdev.azurecr.io imageRepository:app-sre/aro-hcp-clusters-service insecure:false jwksUrl:http://localhost logLevel:4 machinePoolMigrationWorkerPeriod:1h managedIdentitiesDataPlaneAudienceResource:https://dummy.org maximumExpiration:0 memoryLimit:unlimited memoryRequest:150Mi ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev ocpAcrResourceUrl:arohcpocpdev.azurecr.io oidcIssuerBaseUrl:https://arohcpci01oidcj7660416.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpci01oidcj7660416.blob.core.windows.net/ postgresqlVersion:12 provisionShardClusterLimit:100 pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:app-sre/aro-hcp-clusters-service:pull workloadIdentityClientId:0531f05d-8afc-4435-86c7-b9a82d26a229 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] region:westus3 replicas:3 runtimeMode:aro-hcp serviceAccountName:clusters-service serviceKeyVaultName:aro-hcp-dev-svc-kv tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesEnabled:true tlsCertificatesIssuer:Self tokenUrl:http://localhost useAzureDB:false userDefinedDnsBaseDomain:i1.devshift.org]\"\ntime=2026-10-16T16:06:46.344Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-16T16:06:46.586Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy namespace=clusters-service\ntime=2026-10-16T16:06:46.981Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy dryRun=false\ntime=2026-10-16T16:06:46.981Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-16T16:06:47.117Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-16T16:16:56.303Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-10-16T16:16:56.304Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7512/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111107720617660416","occurred_at":"2026-10-16T14:51:17Z","signature_id":"f7a2f83e4294c3ba837340ee15ecf8b39ec31b12220850df230141abca88bbe0","pr_number":7512,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"9edb2d39bdff9cf1586610e7e1778893f0414c6b0749314791e92df31b54ec6f","failure_pattern":"failed to roll out Helm release: resource Deployment/maestro/maestro not ready. status: Failed, message: Progress deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro not ready. status: Failed, message: Progress deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Server/service/deploy","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Server/service/deploy","occurrences":1}],"full_error_samples":["time=2026-10-13T18:29:04.438Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-13T18:29:04.984Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy values=\"map[azure:map[clientId:e73242a2-1d48-4914-b494-303251483672 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] backend:map[namespace:aro-hcp serviceAccount:backend] broker:map[host:arohcp-ci01-maestro-j4029440.westus3-1.ts.eventgrid.azure.net port:8883] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:maestro-server-j4029440] database:map[azureDb:false containerizedDbImage:docker.io/library/postgres:14.2 containerizedDbPvcCapacity:512Mi debug:false host:maestro-db maxOpenConnections:50 name:maestro password:TheBlurstOfTimes port:5432 user:maestro-server] deployment:map[replicas:2 resources:map[limits:map[memory:unlimited] requests:map[cpu:200m memory:512Mi]] zoneCount:3] fleet:map[namespace:fleet serviceAccount:fleet] image:map[digest:sha256:ccc6c4381bbbc6a7af65d4c3886e20baea76f7bb95ea6048b33ee31e8046e5cb registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] istio:map[restrictIngress:true] maestro:map[glog_v:2 grpcBindPort:8090 healthCheckBindPort:8083 httpBindPort:8000 httpReadTimeout:60s httpWriteTimeout:60s labelMetricsInclusionDuration:168h serviceAccount:maestro] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro:pull workloadIdentityClientId:1176baba-6cca-46e9-86bb-3a9c587b6d0c workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX]]\"\ntime=2026-10-13T18:29:04.984Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-10-13T18:29:05.371Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy namespace=maestro\ntime=2026-10-13T18:29:05.621Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy dryRun=false\ntime=2026-10-13T18:29:05.621Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-10-13T18:29:05.751Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-10-13T18:39:13.113Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy err=\"resource Deployment/maestro/maestro not ready. status: Failed, message: Progress deadline exceeded\"\ntime=2026-10-13T18:39:13.113Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro not ready. status: Failed, message: Progress deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110042774484029440","occurred_at":"2026-10-13T16:19:32Z","signature_id":"2a551764d2de0ec2ce5493d79b0c6e2957d9ec9ebccce1e0772fad9dff29448f","pr_number":7502,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-10-05"],"prior_runs_affected":19,"prior_last_seen_at":"2026-10-09T02:34:36Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1}],"full_error_samples":["time=2026-10-15T16:33:01.433Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-15T16:33:03.008Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-15T16:33:06.001Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=4a75caebbcb0957378a554574a53ab5c38ad767d8dfe35c65ac1690bd2903591 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5342592-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F4a75caebbcb0957378a554574a53ab5c38ad767d8dfe35c65ac1690bd2903591\ntime=2026-10-15T17:03:01.188Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110762016095342592","occurred_at":"2026-10-15T15:57:35Z","signature_id":"546713f253dbb07bdf6d6b240d468c73b1927023c4bbf21dee04e87aaa304e9a","pr_number":6746,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"0cd69af8904472f73d329003c8c5db9d0e892958b3af0d73d1a1dee951ccff6d","failure_pattern":"scheduling requested CPU must be positive","search_query":"stamp 1 scheduling requested CPU must be positive","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Fleet should have scheduling data for ready management clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-22T12:46:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Fleet should have scheduling data for ready management clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.000s.\nscheduling data was not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:156 with:\nstamp 1 scheduling requested CPU must be positive\nExpected\n    \u003cbool\u003e: true\nto be false\nfail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.000s.\nscheduling data was not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:156 with:\nstamp 1 scheduling requested CPU must be positive\nExpected\n    \u003cbool\u003e: true\nto be false"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","signature_id":"0dd2468d8f40ede925773d79aa00d7b2731ac739d0aec50490727e167d99e1ce","pr_number":6982,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"69cfaa9caed7e22dd6863796a2f42fb482942befe7200b2f0caeda831337f912","failure_pattern":"waiting for a presented DNS-01 challenge for *","search_query":"waiting for a presented DNS-01 challenge for *","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.20366598778004075,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-11..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:444]: Timed out after 90.001s.\ndelegated NS and ACME TXT should resolve consistently on all authoritative nameservers, or the certificate should already be issued\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2cc845a2b820\u003e: \n    waiting for a presented DNS-01 challenge for *.apps.aro.ingress-cert-le.p7kd.j2563328.hcp.osadev.cloud\n    {\n        s: \"waiting for a presented DNS-01 challenge for *.apps.aro.ingress-cert-le.p7kd.j2563328.hcp.osadev.cloud\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:444]: Timed out after 90.001s.\ndelegated NS and ACME TXT should resolve consistently on all authoritative nameservers, or the certificate should already be issued\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2cc845a2b820\u003e: \n    waiting for a presented DNS-01 challenge for *.apps.aro.ingress-cert-le.p7kd.j2563328.hcp.osadev.cloud\n    {\n        s: \"waiting for a presented DNS-01 challenge for *.apps.aro.ingress-cert-le.p7kd.j2563328.hcp.osadev.cloud\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7416/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110506304622563328","occurred_at":"2026-10-14T23:01:28Z","signature_id":"c83040805ddaac240d23b90d241ffdf180ac51f18dd8ed271efd3fd79b2f2191","pr_number":7416,"after_last_push_of_merged_pr":false}]}]}]}